diff --git a/.gitattributes b/.gitattributes index 14979ec3313f7f843d5ac3f736992f65b12df522..19503c8a7d3b981db93df8beb4f3d73a172fe0e8 100644 --- a/.gitattributes +++ b/.gitattributes @@ -129,3 +129,119 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text 0219133414_cosmos_tnt_object/videos/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.mp4 filter=lfs diff=lfs merge=lfs -text 0219133414_cosmos_tnt_object/videos/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.mp4 filter=lfs diff=lfs merge=lfs -text 0219133414_cosmos_tnt_object/videos/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef.mp4 filter=lfs diff=lfs merge=lfs -text +0219155836_cosmos_tnt_spatial/videos/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.lock new file mode 100644 index 0000000000000000000000000000000000000000..e35a3c106c7f9390b67491ccca20471c1f31731e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542319 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.lock new file mode 100644 index 0000000000000000000000000000000000000000..984355579f0f2d06685e1055330354be65a2595f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542320 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.lock new file mode 100644 index 0000000000000000000000000000000000000000..599dd8aca4899b9d55bab12358d6041b73f8f3d9 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542322 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d.lock new file mode 100644 index 0000000000000000000000000000000000000000..984355579f0f2d06685e1055330354be65a2595f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542320 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.lock new file mode 100644 index 0000000000000000000000000000000000000000..1f47fc3e5111ae355eb5b8fc78649b01f220d476 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542321 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.lock new file mode 100644 index 0000000000000000000000000000000000000000..984355579f0f2d06685e1055330354be65a2595f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542320 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.lock new file mode 100644 index 0000000000000000000000000000000000000000..1f47fc3e5111ae355eb5b8fc78649b01f220d476 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542321 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904.lock new file mode 100644 index 0000000000000000000000000000000000000000..c1806e3b637526e85cf5121c389d832ec25b2688 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542317 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.lock new file mode 100644 index 0000000000000000000000000000000000000000..ce2a9b4d262f6a96bd6f074f8bd241e02e1f1b6e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542323 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.lock new file mode 100644 index 0000000000000000000000000000000000000000..599dd8aca4899b9d55bab12358d6041b73f8f3d9 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542322 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.lock new file mode 100644 index 0000000000000000000000000000000000000000..de21962f561e425615f646850283a381181d3b98 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542318 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0.lock new file mode 100644 index 0000000000000000000000000000000000000000..e35a3c106c7f9390b67491ccca20471c1f31731e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542319 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.lock new file mode 100644 index 0000000000000000000000000000000000000000..de21962f561e425615f646850283a381181d3b98 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542318 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.lock new file mode 100644 index 0000000000000000000000000000000000000000..c1806e3b637526e85cf5121c389d832ec25b2688 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542317 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.lock new file mode 100644 index 0000000000000000000000000000000000000000..e35a3c106c7f9390b67491ccca20471c1f31731e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542319 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.lock new file mode 100644 index 0000000000000000000000000000000000000000..ce2a9b4d262f6a96bd6f074f8bd241e02e1f1b6e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542323 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a.lock new file mode 100644 index 0000000000000000000000000000000000000000..599dd8aca4899b9d55bab12358d6041b73f8f3d9 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542322 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.lock new file mode 100644 index 0000000000000000000000000000000000000000..1f47fc3e5111ae355eb5b8fc78649b01f220d476 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542321 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef.lock new file mode 100644 index 0000000000000000000000000000000000000000..de21962f561e425615f646850283a381181d3b98 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542318 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2.lock b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2.lock new file mode 100644 index 0000000000000000000000000000000000000000..ce2a9b4d262f6a96bd6f074f8bd241e02e1f1b6e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/.locks/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2542323 \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/gpu_0.log b/0219155836_cosmos_tnt_spatial/gpu_0.log new file mode 100644 index 0000000000000000000000000000000000000000..e8a5333f4f3de5a214dbf609f870e9a9a3f7d178 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_0.log @@ -0,0 +1,6 @@ +[lock] Claimed: mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f (worker=bravo-tongclass-a800:2542323) +[lock] Done: mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f +[lock] Claimed: mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144 (worker=bravo-tongclass-a800:2542323) +[lock] Done: mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144 +[lock] Claimed: mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2 (worker=bravo-tongclass-a800:2542323) +[lock] Done: mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2 diff --git a/0219155836_cosmos_tnt_spatial/gpu_0_instances.txt b/0219155836_cosmos_tnt_spatial/gpu_0_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..c5afea04116975ab672ddaf6da164e0c981ae5b2 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_0_instances.txt @@ -0,0 +1,3 @@ +mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f +mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144 +mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2 diff --git a/0219155836_cosmos_tnt_spatial/gpu_1.log b/0219155836_cosmos_tnt_spatial/gpu_1.log new file mode 100644 index 0000000000000000000000000000000000000000..9cb6485650c9121ff1d53dc1860ec280d73df87e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_1.log @@ -0,0 +1,6 @@ +[lock] Claimed: mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e (worker=bravo-tongclass-a800:2542320) +[lock] Done: mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e +[lock] Claimed: mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0 (worker=bravo-tongclass-a800:2542320) +[lock] Done: mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0 +[lock] Claimed: mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d (worker=bravo-tongclass-a800:2542320) +[lock] Done: mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d diff --git a/0219155836_cosmos_tnt_spatial/gpu_1_instances.txt b/0219155836_cosmos_tnt_spatial/gpu_1_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..d7ceba27ecaf6bd2c63e8df212f50f013687f716 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_1_instances.txt @@ -0,0 +1,3 @@ +mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e +mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0 +mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d diff --git a/0219155836_cosmos_tnt_spatial/gpu_2.log b/0219155836_cosmos_tnt_spatial/gpu_2.log new file mode 100644 index 0000000000000000000000000000000000000000..11cba21066faf6b76955455baf3647fd5946d7ea --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_2.log @@ -0,0 +1,6 @@ +[lock] Claimed: mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017 (worker=bravo-tongclass-a800:2542321) +[lock] Done: mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017 +[lock] Claimed: mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857 (worker=bravo-tongclass-a800:2542321) +[lock] Done: mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857 +[lock] Claimed: mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1 (worker=bravo-tongclass-a800:2542321) +[lock] Done: mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1 diff --git a/0219155836_cosmos_tnt_spatial/gpu_2_instances.txt b/0219155836_cosmos_tnt_spatial/gpu_2_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..fdb775d74a8178d4dccbf9abbf35050d5faef7b9 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_2_instances.txt @@ -0,0 +1,3 @@ +mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017 +mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857 +mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1 diff --git a/0219155836_cosmos_tnt_spatial/gpu_3.log b/0219155836_cosmos_tnt_spatial/gpu_3.log new file mode 100644 index 0000000000000000000000000000000000000000..6fb52bf3cf0cbff13a276395c1e4751f949aa5d6 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_3.log @@ -0,0 +1,6 @@ +[lock] Claimed: mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc (worker=bravo-tongclass-a800:2542322) +[lock] Done: mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc +[lock] Claimed: mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1 (worker=bravo-tongclass-a800:2542322) +[lock] Done: mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1 +[lock] Claimed: mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a (worker=bravo-tongclass-a800:2542322) +[lock] Done: mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a diff --git a/0219155836_cosmos_tnt_spatial/gpu_3_instances.txt b/0219155836_cosmos_tnt_spatial/gpu_3_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..2afede437e17b901f16671b5b365ee927871f332 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_3_instances.txt @@ -0,0 +1,3 @@ +mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc +mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1 +mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a diff --git a/0219155836_cosmos_tnt_spatial/gpu_4.log b/0219155836_cosmos_tnt_spatial/gpu_4.log new file mode 100644 index 0000000000000000000000000000000000000000..f4c8a00ca9bf646a05aaf8ddc0b36aa49994c75d --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_4.log @@ -0,0 +1,6 @@ +[lock] Claimed: mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f (worker=bravo-tongclass-a800:2542319) +[lock] Done: mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f +[lock] Claimed: mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5 (worker=bravo-tongclass-a800:2542319) +[lock] Done: mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5 +[lock] Claimed: mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0 (worker=bravo-tongclass-a800:2542319) +[lock] Done: mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0 diff --git a/0219155836_cosmos_tnt_spatial/gpu_4_instances.txt b/0219155836_cosmos_tnt_spatial/gpu_4_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..f0ffcaa9b2cf3cc91d200097909be534c9497372 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_4_instances.txt @@ -0,0 +1,3 @@ +mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f +mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5 +mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0 diff --git a/0219155836_cosmos_tnt_spatial/gpu_5.log b/0219155836_cosmos_tnt_spatial/gpu_5.log new file mode 100644 index 0000000000000000000000000000000000000000..6b77b2adf3ec734d860985cdb5eb2ee5b55932bd --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_5.log @@ -0,0 +1,6 @@ +[lock] Claimed: mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21 (worker=bravo-tongclass-a800:2542318) +[lock] Done: mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21 +[lock] Claimed: mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de (worker=bravo-tongclass-a800:2542318) +[lock] Done: mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de +[lock] Claimed: mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef (worker=bravo-tongclass-a800:2542318) +[lock] Done: mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef diff --git a/0219155836_cosmos_tnt_spatial/gpu_5_instances.txt b/0219155836_cosmos_tnt_spatial/gpu_5_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..b440c5f83c33be764b4e1edc397a1edbbd305f40 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_5_instances.txt @@ -0,0 +1,3 @@ +mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21 +mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de +mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef diff --git a/0219155836_cosmos_tnt_spatial/gpu_6.log b/0219155836_cosmos_tnt_spatial/gpu_6.log new file mode 100644 index 0000000000000000000000000000000000000000..d17f7a954b924ce8b1893fff7da06551e0015b86 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_6.log @@ -0,0 +1,4 @@ +[lock] Claimed: mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b (worker=bravo-tongclass-a800:2542317) +[lock] Done: mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b +[lock] Claimed: mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904 (worker=bravo-tongclass-a800:2542317) +[lock] Done: mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904 diff --git a/0219155836_cosmos_tnt_spatial/gpu_6_instances.txt b/0219155836_cosmos_tnt_spatial/gpu_6_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..1048d13346c16cdaeb54669dea67c68b9c73b880 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/gpu_6_instances.txt @@ -0,0 +1,2 @@ +mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b +mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904 diff --git a/0219155836_cosmos_tnt_spatial/mapping.json b/0219155836_cosmos_tnt_spatial/mapping.json new file mode 100644 index 0000000000000000000000000000000000000000..8fbde9b091edafb5341427526a4d7c9d5922c81b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mapping.json @@ -0,0 +1,586 @@ +[ + { + "instance_id": "mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + } +] \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7f4d57f187ac94b579b44db2ecf7d8516a78e3a7 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/console.log new file mode 100644 index 0000000000000000000000000000000000000000..ea046d17743c8444cadbd2f754418c1df3bc315b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/console.log @@ -0,0 +1,22 @@ +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/config.yaml +[02-19 18:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.json +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:32:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:32:24|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a1da5c66bbbfa835d529f87da44caacb5bb29fc0 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/debug.log @@ -0,0 +1,353 @@ +[02-19 18:17:12|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 18:17:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:17:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:17:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 18:17:21|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 18:17:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 18:17:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 18:17:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 18:17:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 18:17:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 18:17:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 18:17:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 18:17:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 18:17:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 18:17:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 18:17:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 18:17:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 18:17:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 18:17:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 18:17:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 18:17:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 18:17:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 18:17:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 18:17:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 18:17:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 18:17:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 18:17:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 18:17:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 18:17:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 18:17:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 18:17:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 18:17:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 18:17:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 18:17:25|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 18:17:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 18:17:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:17:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:17:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 18:17:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:17:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 18:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:17:27|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 18:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 18:17:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:17:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:17:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:17:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:17:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 18:17:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 18:17:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.45 s +[02-19 18:17:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:18:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:18:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 18:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/config.yaml +[02-19 18:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:18:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="80-100%: In the final segment, the camera provides a panoramic view of the Panther tank '501' from multiple angles, summarizing the earlier detailed shots. It pans slowly around the tank once more, ensuring all key features are captured. The video concludes with a wide shot that encompasses the entire tank and its surroundings, leaving a lasting impression of the vehicle's imposing presence in the hangar." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 18:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.json +[02-19 18:18:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:18:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4 +[02-19 18:18:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 18:18:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 18:18:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 18:18:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 18:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 18:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:32:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:32:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 18:32:24|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..8821e0c6716fbd7f6f9ba7743c5471f98452d5f4 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"0-20%: The video opens with a close-up shot of the Panther tank '501', focusing on its front-left side. The camera captures the detailed camouflage pattern, the number '501' on the turret, and the initial road wheels. The open hangar bay door is visible in the background, along with parts of other tanks lined up nearby.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..164bd1ae815896512a58d178c84e0b29b0311346 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0aa5adb01742d65828a6bbf982fe1c2ea89861ea03e861348bd2779178574b3d +size 973815 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..f6635ddb56fdf769f934ff31944c90bf0827e62f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"20-40%: The camera begins a smooth horizontal pan to the right, circling the tank and revealing its side profile. As it moves, the camera tilts slightly upward to focus on the turret and gun barrel, then zooms out to provide a broader view of the tank's length and its position within the hangar. The background remains consistent, showing the open bay door and adjacent tanks.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3b1584bc7501a3577b0af39f6e5e271973119c5b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3aa98ff9f4a6e58bf6def95a27d3528d7387471295806034002072a7efeb7366 +size 1205219 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..fcbd459277a00243aa239f81ac4ca092f9740b8a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"40-60%: Continuing its circular motion, the camera moves toward the rear of the Panther tank, capturing the back panel and exhaust pipes. It then tilts downward to focus on the tracks and road wheels, zooming in for a detailed look at these components. The camera retraces part of its earlier path, revisiting the side and front views of the tank while maintaining spatial consistency with the environment.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f825839277904a81fa69000e43cda571ace11dc7 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e630c950186c8455365c1c6a0f8241dd3c059142ce920bda142ea39bad9639c +size 1396240 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..4aeb91068b1c5e57c34f93a96841e5588bf3200e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"60-80%: The camera completes its circle around the tank, returning to the front-left side. It zooms in again on the turret and gun barrel, emphasizing the number '501' and the intricate details of the tank's exterior. The open hangar door and other tanks in the background reappear, reinforcing the spatial context. The camera then begins to pull back, preparing for the final overview.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6553e2d176d2dfb2adf96d8f47621a8312661a44 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58a3e23dbef9306585eb4aabacbde8da563dfb0a36439e6e316dcdc76308a482 +size 1638414 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..eae8ce1c6c66a7e08cf58ae7ee822110fdf2cfc6 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"80-100%: In the final segment, the camera provides a panoramic view of the Panther tank '501' from multiple angles, summarizing the earlier detailed shots. It pans slowly around the tank once more, ensuring all key features are captured. The video concludes with a wide shot that encompasses the entire tank and its surroundings, leaving a lasting impression of the vehicle's imposing presence in the hangar.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..534972cf7b22f304a4c4135456cf08102f0e5967 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2de73ba5a0cb69a567881a7169a63c5af60e62b4c77e2ae287091a2347c9dad +size 1680726 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e2d94125de0d09abdc4526b02f16c7c859d75ef --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/console.log new file mode 100644 index 0000000000000000000000000000000000000000..01680798faafba5aed65700794ceaf37f5c49e42 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/console.log @@ -0,0 +1,22 @@ +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:09:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:09:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:09:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:09:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:09:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/config.yaml +[02-19 18:09:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:09:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:09:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.json +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:09:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:23:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:23:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:23:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:23:27|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e060864939a4a8699a2c63969af792be1f9edab9 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/debug.log @@ -0,0 +1,353 @@ +[02-19 18:09:11|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 18:09:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:09:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:09:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 18:09:21|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 18:09:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 18:09:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 18:09:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 18:09:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 18:09:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 18:09:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 18:09:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 18:09:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 18:09:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 18:09:21|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 18:09:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 18:09:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 18:09:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 18:09:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 18:09:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 18:09:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 18:09:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 18:09:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 18:09:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 18:09:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 18:09:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 18:09:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 18:09:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 18:09:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 18:09:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 18:09:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 18:09:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 18:09:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 18:09:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 18:09:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 18:09:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:09:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:09:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 18:09:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:09:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:09:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:09:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 18:09:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:09:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 18:09:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:09:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:09:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 18:09:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:09:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.07 s +[02-19 18:09:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 18:09:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 18:09:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 18:09:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:09:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 18:09:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 18:09:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 18:09:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:09:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:09:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 18:09:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 18:09:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.87 s +[02-19 18:09:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:09:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 18:09:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:09:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/config.yaml +[02-19 18:09:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:09:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:09:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="80-100%: In the final segment, the camera retraces its earlier paths, revisiting the audience area and the judge's bench. It captures the consistent positioning of the American flag and the portraits on the walls. The video concludes with a final view of the stained-glass dome, emphasizing its beauty and the overall grandeur of the courtroom." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 18:09:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.json +[02-19 18:09:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:09:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:09:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4 +[02-19 18:10:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 18:10:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 18:10:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 18:10:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 18:23:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 18:23:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:23:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:23:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:23:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 18:23:27|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..51197d161b17b2bb7257233ee5a77b2da4680cf6 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"0-20%: The video opens with a sweeping view of the courtroom's audience area, showcasing the rows of wooden chairs with teal cushions and the red patterned carpet. The camera slowly pans upward, revealing the ornate ceiling with its detailed moldings and the central stained-glass dome. Key spatial anchors include the American flag near the judge's bench and the large windows with blinds along the walls.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2656220322bb2d6ffe193e6f43c5e4ad8089f5a2 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:391a6701c1952a5725026d1d4d584e2d5307b24e81db9fe3cce85e4435978367 +size 1343316 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..1a0352c06410f821141a89d339ede5ace6a7602e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"20-40%: The camera continues its exploration by moving closer to the judge's bench and the jury box, capturing the rich woodwork and the formal setup of the courtroom. It then tilts upward to focus on the stained-glass dome, emphasizing its vibrant colors and intricate designs. The camera retraces its path, returning to the audience area and maintaining consistent lighting and object positions.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d8a9c21796f425d2ee7f5e1e119e8956c1dd8c7a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:52a3a444a51b3c5b3ce2af61769c49f6c938037f7fa5a7ec53c6085d9a7aef77 +size 1384926 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..07183f7e18f25fbcf9d820782d4d19d9d09a4acc --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"40-60%: As the video progresses, the camera moves horizontally around the courtroom, capturing different angles of the wooden furnishings and the architectural details. It revisits the stained-glass dome from various perspectives, highlighting the interplay of light and color. The camera also captures the re-entry of background elements, such as the windows and portraits, ensuring their consistent appearance.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c6d7f29f59fdbc5f60ab3ff654904afec375ba2a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:927bd5a43e364501c3353ec240641a1a06f52b8737d8c432da95a151e4a2efde +size 1281102 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..b93016c8b0fcc8aece39f7bf50b2613587c223ff --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"60-80%: The focus shifts to the vertical perspective changes, with the camera looking directly up at the dome and then tilting down to show the courtroom floor. This transition is smooth, maintaining the spatial coherence of the environment. The video then moves to the side of the courtroom, capturing the jury box and the additional seating areas, before returning to the front of the room.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..040b7d73e49788cc6d45fecb81d1eeaf0a0c3520 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff2b1ede7be0385faa88afc5aec39651fccddae40218979cfb1d03c7c8f440a6 +size 1408968 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..5c96ae604fb1501544105a35589436c1bcfd4a0a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"80-100%: In the final segment, the camera retraces its earlier paths, revisiting the audience area and the judge's bench. It captures the consistent positioning of the American flag and the portraits on the walls. The video concludes with a final view of the stained-glass dome, emphasizing its beauty and the overall grandeur of the courtroom.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e25a28bd9f048cb4432911f5157c69ec90499c83 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3565f16cb1450db63844fb7879fd9e83330fdf3065219799eeba221f23a8bee3 +size 1955199 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60b0ec6dd61dbe130d9f68f4563702ec31f4388a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/console.log new file mode 100644 index 0000000000000000000000000000000000000000..f58955d59b41acc6fbdde8e359e0be2b2a144159 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/console.log @@ -0,0 +1,22 @@ +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:08:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:08:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/config.yaml +[02-19 18:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_4.json +[02-19 18:08:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:22:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:22:05|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..dd73b210704f7678c6699773779e85d438f07bd7 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/debug.log @@ -0,0 +1,353 @@ +[02-19 18:07:54|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 18:08:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:08:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:08:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 18:08:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 18:08:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 18:08:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 18:08:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 18:08:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 18:08:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 18:08:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 18:08:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 18:08:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 18:08:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 18:08:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 18:08:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 18:08:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 18:08:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 18:08:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 18:08:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 18:08:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 18:08:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 18:08:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 18:08:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 18:08:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 18:08:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 18:08:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 18:08:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 18:08:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 18:08:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 18:08:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 18:08:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 18:08:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 18:08:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 18:08:07|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 18:08:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 18:08:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:08:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:08:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 18:08:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:08:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:08:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:08:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 18:08:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:08:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 18:08:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:08:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:08:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 18:08:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:08:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.23 s +[02-19 18:08:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 18:08:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 18:08:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 18:08:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 18:08:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 18:08:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 18:08:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:08:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:08:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:08:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 18:08:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 18:08:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.26 s +[02-19 18:08:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:08:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 18:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/config.yaml +[02-19 18:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:08:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="80-100%: In the final segment, the camera makes minor adjustments, slightly zooming in and out while maintaining the frontal view. The video concludes by reinforcing the tank's imposing presence within the hangar, with the background tanks and open door providing a sense of depth and continuity. The lighting remains consistent, highlighting the tank's weathered surface and intricate details." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 18:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_4.json +[02-19 18:08:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:08:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_3.mp4 +[02-19 18:08:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 18:08:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 18:08:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 18:08:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 18:22:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 18:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:22:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:22:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:22:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 18:22:05|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..0dc205ea39d4dc2a2739e75a6f2ec5cb09f8a6fe --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"0-20%: The video opens with a close-up frontal view of the tank, emphasizing its rugged front grille and identification markers. The camera is steady, allowing viewers to absorb the details of the tank's front armor and track system. The red-painted interior of the hangar and the bright light from the open garage door are visible in the background.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f60107f7fa30926a0328dcc0c4e02021d94ab71e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ccd4cabead2f6b8b1fc40836a07c38520bac725e8097b4c91ecb3765dad0554c +size 924753 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..68ca49c29348cccd6395946c7d3b2c5ebb8e86b9 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"20-40%: The camera begins a smooth pan to the right, tracking along the side of the tank. As it moves, the focus shifts to the tank's tracks, road wheels, and the turret, which is equipped with mounted equipment and supplies. The open garage door comes into view, revealing an outdoor setting with trees and a parked car. The camera maintains a consistent distance, providing a continuous side profile of the tank.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b893168b73d3dad9ce9d6d86fdef5895d69ae8ff --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4eafb3dd1a3b36c439bd9b3b60181f6a6caeb89785e05ab28dc56c97c4106196 +size 1250241 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..262f9153ce7c4e46d1fc1b59b063d2ee9c228106 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"40-60%: Continuing its orbit, the camera moves further around the tank, capturing its rear and top sections. The turret's details, including hatches and external fittings, become more prominent. The camera then starts to retrace its path, moving back toward the front of the tank. The background tanks and the open garage door remain visible, reinforcing the spatial context of the hangar environment.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..335f0dc7e6ac803267e3ea479d5ace3b92ed06ec --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bb0e73c5cc3ade1869f890ba6eecb407368fdacc0610376593737253797249e +size 1578540 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..4039272faaa53351ea58b077edfe8c3b0cc77490 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"60-80%: The camera completes its circular motion, returning to a frontal perspective similar to the opening shot. The focus remains on the tank's front grille and overall structure, with the open garage door still visible in the background. The video emphasizes the tank's symmetrical design and the consistency of its features from different angles.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f2ccbf6e25b3fc8197d7a155e031bb162a4844ad --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34f89205c061341453b20c754306a19a60178244a461ef85c7e76e001f17ec9f +size 1549306 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..5753f49a2562368337cce23200f441000c5cc483 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"80-100%: In the final segment, the camera makes minor adjustments, slightly zooming in and out while maintaining the frontal view. The video concludes by reinforcing the tank's imposing presence within the hangar, with the background tanks and open door providing a sense of depth and continuity. The lighting remains consistent, highlighting the tank's weathered surface and intricate details.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bfada3c3fb595fd0407a3b3d54f418d9836ee5a1 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6c7194c67ed777e7241dfbb4924228fa885293f053f3229e2cd6e5d4c102574 +size 1074778 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af5e2688b090bb7c0848db04882165e6b139fe70 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/console.log new file mode 100644 index 0000000000000000000000000000000000000000..04c3117dec4904cf3e0fe6e336cdadfb653a538b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/console.log @@ -0,0 +1,22 @@ +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:21:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:21:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:22:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/config.yaml +[02-19 19:22:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 19:22:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 19:22:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_4.json +[02-19 19:22:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:22:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 19:35:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 19:35:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 19:35:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 19:35:34|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c18a47b2b71972c212118f90c3f3f5c28da4ef05 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/debug.log @@ -0,0 +1,353 @@ +[02-19 19:21:20|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 19:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:21:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:21:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 19:21:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 19:21:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 19:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 19:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 19:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 19:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 19:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 19:21:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 19:21:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 19:21:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 19:21:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 19:21:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 19:21:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 19:21:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 19:21:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 19:21:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 19:21:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 19:21:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 19:21:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 19:21:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 19:21:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 19:21:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 19:21:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 19:21:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 19:21:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 19:21:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 19:21:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 19:21:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 19:21:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 19:21:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 19:21:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 19:21:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:21:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:21:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 19:21:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:21:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:21:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:21:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 19:21:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:21:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 19:21:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:21:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 19:21:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:21:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:22:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 19:22:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 19:22:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.01 s +[02-19 19:22:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:22:06|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:22:07|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 19:22:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:22:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/config.yaml +[02-19 19:22:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 19:22:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 19:22:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='The final segment of the video provides a comprehensive overview of the Herbert Maier Memorial Auditorium, combining all the previously explored perspectives into a cohesive narrative. The camera moves smoothly around the building, capturing the full scope of its design and the surrounding park setting. It concludes with a wide shot that encompasses the entire structure, from the front entrance to the back colonnade, and includes the park elements that frame it. The lighting remains consistent throughout, ensuring a seamless transition between different parts of the video and reinforcing the spatial and environmental continuity.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 19:22:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_4.json +[02-19 19:22:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:22:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 19:22:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_3.mp4 +[02-19 19:22:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 19:22:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 19:22:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 19:22:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 19:35:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 19:35:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 19:35:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 19:35:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 19:35:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 19:35:34|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..a9d9840d9cbf58b6785c961f9ef2f98b4b675802 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a sweeping view of the Herbert Maier Memorial Auditorium, focusing on the front entrance characterized by a series of tall, evenly spaced columns supporting a decorative balustrade. The camera slowly pans to the left, revealing the semi-circular stage area with its intricately designed dome ceiling. In the background, park elements such as trees and pathways are visible, framing the structure and adding depth to the scene. The lighting is soft, suggesting an overcast day, which enhances the textures of the stone architecture.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2967bada06402cf5f678d576f4f9232ba1d40b6b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81537392d42d6fe42aeb1cfa165caf54c7ea4280aebdb5a6f531e9828464233e +size 933912 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..b13cac3bacc8691731b0a9dcfe3bf16d3344f747 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues its movement, it shifts focus to the right side of the auditorium, capturing the continuation of the colonnade and the detailed balustrade. The perspective changes slightly, offering a closer look at the ornate carvings above the archway and the robustness of the columns. The camera then tilts upward, emphasizing the grandeur of the dome and the intricate patterns within it. Meanwhile, the park setting remains consistent, with trees and pathways subtly entering and exiting the frame, maintaining spatial continuity.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..01a29eba8dac6aa91ab3740b48bdafc2f5b610d2 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be687cc6781635c3e617712137a2514c9cefd9e5d387007d73956b1a8638455e +size 938759 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..dc57652ed7345c1b82f4f0e516549db02c9b571f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The middle section of the video delves deeper into the architectural details of the Herbert Maier Memorial Auditorium. The camera moves along the colonnade, providing close-up views of the columns' capitals and the textured surfaces of the stone. It then transitions to a lower perspective, looking up at the dome and capturing the interplay of light and shadow across its surface. The surrounding park elements, including benches and flowerbeds, are intermittently visible, reinforcing the connection between the building and its environment. The camera then begins to retrace its path, returning to earlier viewpoints to ensure spatial coherence.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f9de1c306a87e53aba0e58865e28d63ee50978a2 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:308cffc531f0b69455683e1046439dda988c57a8ef6dadb4945a62bbd24d9624 +size 1606321 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..a87b6df4ce04c69ab8d7967286fd34d597b67325 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the fourth segment, the camera retraces its path around the auditorium, revisiting the front entrance and the semi-circular stage area. This time, however, the perspective is slightly altered, offering a fresh angle on the same features. The camera pans back to the left, capturing the colonnade once again and emphasizing the symmetry of the design. The background elements, such as the trees and pathways, reappear in the frame, maintaining the spatial relationship established earlier. The video then shifts focus to the steps leading up to the stage, highlighting their texture and the metal railings that flank them.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..db8b13347de9e2e813c89dacde17fef891b724fc --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63e56994cf0b24323311af40d69ed94790a7918630ca7e492ac0f4b971d77c48 +size 1930420 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..891391343de00669a32047d3053b15f7c37f924f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The final segment of the video provides a comprehensive overview of the Herbert Maier Memorial Auditorium, combining all the previously explored perspectives into a cohesive narrative. The camera moves smoothly around the building, capturing the full scope of its design and the surrounding park setting. It concludes with a wide shot that encompasses the entire structure, from the front entrance to the back colonnade, and includes the park elements that frame it. The lighting remains consistent throughout, ensuring a seamless transition between different parts of the video and reinforcing the spatial and environmental continuity.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..65fba726900e0002698a3795b558e25a555cb104 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e2bce037c5c4360c834fbfefae9c94f2c0d29bc9578d61445c4fa79ed93c29f +size 1907301 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..495e869b7946d0386f069fe9cb73ca1d99a3ff3d --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/console.log new file mode 100644 index 0000000000000000000000000000000000000000..9153b77fde23a0dcb5f97a771f3caf7de406c521 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/console.log @@ -0,0 +1,22 @@ +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 16:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 16:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:57:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/config.yaml +[02-19 16:57:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 16:57:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 16:57:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.json +[02-19 16:57:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 16:57:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 17:10:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:10:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:10:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:10:46|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4d60e150446d6f01eecdc80a7f11779e7bf1eb70 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/debug.log @@ -0,0 +1,353 @@ +[02-19 16:56:31|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 16:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 16:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 16:56:41|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 16:56:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 16:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 16:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 16:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 16:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 16:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 16:56:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 16:56:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 16:56:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 16:56:41|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 16:56:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 16:56:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 16:56:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 16:56:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 16:56:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 16:56:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 16:56:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 16:56:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 16:56:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 16:56:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 16:56:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 16:56:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 16:56:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 16:56:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 16:56:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 16:56:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 16:56:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 16:56:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 16:56:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 16:56:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 16:56:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 16:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 16:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 16:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 16:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 16:56:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 16:56:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 16:56:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.00 s +[02-19 16:56:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 16:56:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 16:56:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 16:56:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:57:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 16:57:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 16:57:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.09 s +[02-19 16:57:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:57:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:20|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 16:57:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 16:57:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/config.yaml +[02-19 16:57:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 16:57:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 16:57:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='80-100%: In the final segment, the camera continues its horizontal exploration, circling the statue once more. It captures the statue from a variety of angles, including close-ups of its facial features and the texture of its clothing. The surrounding garden, with its green lawn, flowerbeds, and distant building, remains a constant presence, providing a cohesive and immersive visual experience.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 16:57:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.json +[02-19 16:57:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 16:57:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 16:57:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4 +[02-19 16:57:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 16:57:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 16:57:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 16:57:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 17:10:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 17:10:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:10:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:10:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:10:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 17:10:46|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..30efee8b19f9cd27349fa0f06c5a1bd8a0ed56fc --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"0-20%: The video opens with a frontal view of the bronze statue of a seated man reading, situated on a circular stone base. The camera captures the statue's intricate details, with the lush green lawn, vibrant flowerbeds, and a beige building with arched windows forming a picturesque background. The lighting is bright and consistent, emphasizing the serene garden setting.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b8db59681af5a23ffc21918f4737d9538e95c7c9 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fde42842a87827ddd11c8cc95bde68d96c9588c48cb5b5b484251da74dbba237 +size 1222268 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..c9f85f21bdac648c653fdeb5b2275732f29d8a11 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"20-40%: The camera begins its horizontal orbit around the statue, smoothly transitioning from the frontal view to side angles. As it moves, the statue's back and side profiles become visible, revealing more of its sculptural form. The surrounding garden elements, including the flowerbeds and distant building, remain consistent, providing a stable backdrop for the dynamic camera movement.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..11a646f8567c484ca811c7750b47b6af64ec9319 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79c03ba6cec52e56ccf742786fee46572abcf4914edd8ac24f1c26d75f53c4d4 +size 1417815 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..918d5900f3273414d8e15af9d4e04a8e60131f6e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"40-60%: Continuing its orbit, the camera captures the statue from various angles, including close-ups of its seated posture and the book it holds. The circular stone base and the shallow pool of water around it are prominently featured. The camera's movement is fluid, maintaining a steady pace as it explores the statue's details while keeping the background elements in view.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..efa0c9b3dd1c35db77f7aef3f709619dce5dae53 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b160a5436e91ae47a8892868848fd084fa1fc4523c4d442d5a16bda07c71ee78 +size 1192272 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..85ab4df13461f6891005c32b341d70352c67449c --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"60-80%: The camera completes its first full orbit around the statue, returning to a perspective similar to the initial frontal shot. It then begins to retrace its path, revisiting previously shown angles and viewpoints. This retracing emphasizes the loop-like nature of the camera movement, reinforcing spatial coherence and allowing viewers to recognize familiar elements within the scene.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e4a43851d1321c9ebf809c26e0ecc6ebbf51e162 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5224c2fe62264d9446ba88d35bcd45acc10e0bd94cd07916ac72e3ad2b0026c2 +size 1259366 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..3b53ae299d661357c8738735d82c82646333d2ed --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"80-100%: In the final segment, the camera continues its horizontal exploration, circling the statue once more. It captures the statue from a variety of angles, including close-ups of its facial features and the texture of its clothing. The surrounding garden, with its green lawn, flowerbeds, and distant building, remains a constant presence, providing a cohesive and immersive visual experience.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1bdc784ac772c0fb0d2030c97db9950a7d8045ee --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb62c905ea1276c46dc17adff3f7afc10a81019c45993cfb786985396960f37c +size 1960638 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98c869f7f2f86b5f87f5c1810f2f18439a232ae0 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/console.log new file mode 100644 index 0000000000000000000000000000000000000000..77881bc3fdcb90911de4f278be655eea433ab238 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/console.log @@ -0,0 +1,22 @@ +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 16:56:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 16:56:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 16:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 16:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/config.yaml +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.json +[02-19 16:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 17:11:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:11:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:11:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:11:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c7545d13321464aa4733350cbdb15ffc15e762cc --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/debug.log @@ -0,0 +1,353 @@ +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 16:56:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 16:56:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 16:56:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 16:56:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 16:56:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 16:56:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 16:56:52|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 16:56:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 16:56:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 16:56:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 16:56:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 16:56:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 16:56:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 16:56:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 16:56:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 16:56:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 16:56:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 16:56:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 16:56:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 16:56:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 16:56:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 16:56:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 16:56:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 16:56:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 16:56:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 16:56:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 16:56:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 16:56:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 16:56:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 16:56:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 16:56:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 16:56:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 16:56:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 16:56:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 16:56:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 16:56:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 16:56:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 16:56:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.47 s +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:57:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 16:57:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 16:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 1.01 s +[02-19 16:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 1.51 s +[02-19 16:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 1.51 s +[02-19 16:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 1.51 s +[02-19 16:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 16:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 16:57:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 16:57:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 16:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 16:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 30.73 s +[02-19 16:57:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:57:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:34|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 16:57:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/config.yaml +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 16:57:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="80-100%: In the final segment, the camera returns to the front-left side of the statue, completing the full circular orbit. The background once again features the building with arched windows and the neatly maintained lawn. The lighting remains consistent, with the soft glow of late afternoon sunlight highlighting the statue's details. The video concludes by reinforcing the spatial continuity established at the beginning, with all key elements reappearing in their expected positions." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.json +[02-19 16:57:35|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4 +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 16:57:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 16:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 16:57:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 17:11:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 17:11:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:11:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:11:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:11:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 17:11:03|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..e14422604acc9ba35844a364a34e0652c244a202 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"0-20%: The video opens with a close-up shot of the bronze horse statue from its front-left side. The statue is captured in a rearing pose, with its front legs lifted and mane flowing backward. The background features a neatly trimmed lawn, several young trees, and a building with a red-tiled roof and large windows. The lighting is soft, suggesting late afternoon, and the camera begins its slow, circular movement around the statue.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..00ec4e63c3b7e60cf87cc455d03b15dc39cbe5dc --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0925e1f02cf5922920f67361369514858039c6f5d09c5e70ef7b9488642a547a +size 1095853 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..9de9bd8702278a392c775cbe2704858f76cae143 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"20-40%: As the camera continues its orbit, it moves from the front-left toward the rear of the statue, revealing more of the surrounding plaza. The background now includes a street with parked cars and pedestrians walking along the sidewalk. A tall palm tree becomes visible on the right side of the frame, adding to the tropical ambiance. The camera maintains a steady pace, keeping the statue centered while gradually shifting the viewer's perspective.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5f566831d3843aee6a265ee0dea9a350b52117a0 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41098ad2bdaad5c775294785815e340dc5fc5205965f5d9224296ba03748cfd6 +size 1060886 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..d87d674730ab4a30cb0f98fd4ed8ad2dabeb2ecb --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"40-60%: The camera reaches the rear of the statue, providing a clear view of its powerful hindquarters and flowing tail. The background now prominently features a modern building with large glass windows and a sloped roof. The transition from the natural elements (trees and grass) to the architectural structures is smooth, with the camera continuing its circular path without abrupt changes in angle or focus.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0c4c3853a4030b217739d5af5728476354dac791 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6d9533cc868fdf0a1741f43c5dc93e28c661cd5a0350c4fdb71684a27bb2fd6 +size 1454125 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..e9a2567b8ddf415a95229dae903ce230225ede5c --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"60-80%: Moving further around the statue, the camera shifts to the right side, capturing the horse's profile as it faces away from the viewer. The background now includes a streetlamp and a glimpse of the street with moving cars. The camera’s movement remains fluid, maintaining a consistent distance from the statue while gradually revealing new elements of the environment, such as additional palm trees and a parking lot in the distance.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5bb34f2152029042f42f00ca8635292dbab34fb5 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f92a0e644c5e4be974f3fbc16d5d448bb3df2cfdacd1ad620ff42a64d518d431 +size 1714354 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..d41526ca56989d26c45c5195e8e0b0d6ce1f4dcd --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"80-100%: In the final segment, the camera returns to the front-left side of the statue, completing the full circular orbit. The background once again features the building with arched windows and the neatly maintained lawn. The lighting remains consistent, with the soft glow of late afternoon sunlight highlighting the statue's details. The video concludes by reinforcing the spatial continuity established at the beginning, with all key elements reappearing in their expected positions.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4eb0cd084cadad4a86c84bc851686cf863765599 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a76ae654507e0ec0597c9dd48d55f443112784e7d44da83c12945c3cb43e262 +size 2059976 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91b92c01b787fec5c8be80ab38443911e120584e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/console.log new file mode 100644 index 0000000000000000000000000000000000000000..a51d8e2af9dbb23ab05b8b71c44a071fb32a52b6 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/console.log @@ -0,0 +1,22 @@ +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:08:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:08:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/config.yaml +[02-19 18:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.json +[02-19 18:09:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:22:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:22:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f3bb4b737521069ea930e084cc1f953296b05f7e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/debug.log @@ -0,0 +1,353 @@ +[02-19 18:08:29|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 18:08:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:08:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 18:08:38|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 18:08:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 18:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 18:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 18:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 18:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 18:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 18:08:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 18:08:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 18:08:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 18:08:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 18:08:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 18:08:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 18:08:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 18:08:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 18:08:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 18:08:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 18:08:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 18:08:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 18:08:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 18:08:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 18:08:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 18:08:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 18:08:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 18:08:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 18:08:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 18:08:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 18:08:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 18:08:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 18:08:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 18:08:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 18:08:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:08:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:08:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 18:08:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:08:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:08:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:08:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 18:08:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:08:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 18:08:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:08:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:08:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 18:08:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:08:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.65 s +[02-19 18:08:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 18:08:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 18:08:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 18:08:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 18:08:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 18:08:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 18:08:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:08:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.65 s +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.65 s +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.65 s +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:08:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:08:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 18:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 18:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.36 s +[02-19 18:09:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:09:16|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:09:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 18:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/config.yaml +[02-19 18:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:09:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final segment, the camera performs a sweeping pan around the entire circumference of the lighthouse, offering a complete 360-degree view of the structure. The sun casts long shadows across the building, emphasizing its texture and depth. The video concludes with a zoomed-out shot that encompasses the entire lighthouse, the American flag, and the surrounding environment, providing a satisfying sense of closure and spatial coherence.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 18:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.json +[02-19 18:09:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:09:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4 +[02-19 18:09:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 18:09:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 18:09:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 18:09:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 18:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 18:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:22:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:22:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:22:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 18:22:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..445d2b1dad5f37f7c42c6a7367c89362b42b2e60 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a static wide shot of the Santa Cruz Surfing Museum lighthouse, showcasing its red brick facade, barred windows, and the American flag prominently displayed on a tall pole. The camera begins a slow horizontal pan to the left, gradually revealing more of the building's side and the adjacent wooden benches. As the pan continues, the flagpole momentarily obstructs the view before clearing, allowing the camera to capture the full height of the lighthouse and its white lantern room at the top.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9ad7ed23e5015bf1a14ebebb8184e0a1fde23a56 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c8ed8a139dbc2a4bf622d70a4dc05f458d5a5719dc372ed7dd983908573ce90 +size 1150413 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..d42979d6ef4f86f8b3fa559e3bdc468d98907cfd --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"Transitioning into the second segment, the camera tilts upward, focusing on the upper portion of the lighthouse to highlight the intricate details of the lantern room and the solar panels mounted on the roof. The American flag flutters in the breeze, adding a dynamic element to the otherwise static shot. After a brief zoom-in on the flag and lantern room, the camera pans downward, retracing its path along the side of the building and passing by the parked cars in the background parking lot.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..444659c77d72d2f634532737175ad6065eaff09b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df3bdca06eb4b600f0eb9218636e4095e4ba4e3cd72a1877bada2e13bfd80800 +size 1416893 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..872f219acde447a10ced6c1f738886943308b25f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"In the third segment, the camera returns to the ground level, circling around the base of the lighthouse once more. This time, it focuses on the museum entrance, where a sign reading 'Santa Cruz Surfing Museum' is clearly visible. The camera then pans to the left, revealing a small tree growing against the brick wall and casting shadows on the building. As the pan continues, the viewer is treated to a series of close-up shots of the lighthouse's architectural details, including the barred windows and decorative trim.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..30986c6b4c8d184b7d11b6f60101646de9a5fc11 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da2df6a30f4c3a2ae8e70baccb09e55920d8dc997c13b77bb511881f6308b200 +size 1573502 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..00fe529678e085a9f4d6d3f26fa1603526ef694f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Moving into the fourth segment, the camera shifts its focus to the right side of the lighthouse, capturing the expansive coastal view with the ocean stretching to the horizon. The American flag remains a constant presence, waving gently in the breeze. The camera then tilts downward, following the line of the building's roofline and capturing the solar panels installed on the sloped roof. As the tilt concludes, the camera pans back to the left, returning to the front of the lighthouse and the museum entrance.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1af7e548035fe4b4a8def1400989016a8710d28a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c95ff9e544d6c45e3ee3c31bb4e1db68016b76fdfe8d019f14d86dd6e0acbec8 +size 1479165 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..194f2bc425f1a1e96c41304440e0b4326be59a26 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the camera performs a sweeping pan around the entire circumference of the lighthouse, offering a complete 360-degree view of the structure. The sun casts long shadows across the building, emphasizing its texture and depth. The video concludes with a zoomed-out shot that encompasses the entire lighthouse, the American flag, and the surrounding environment, providing a satisfying sense of closure and spatial coherence.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ce5c0a183d7bceef6e0c6b3d9242d44b3e019a3e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:694949cc804b5dd2d2e2b691583591b29b7be0b85b52342b781c8929c499bab4 +size 1280052 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..043750fa0cb829880d24e0b56240313ed4e893a0 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/console.log new file mode 100644 index 0000000000000000000000000000000000000000..62be98a8e36c63d2bddc495bc0965b661443a63c --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/console.log @@ -0,0 +1,22 @@ +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:07:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:07:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:07:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:07:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:08:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/config.yaml +[02-19 18:08:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:08:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:08:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_4.json +[02-19 18:08:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:08:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:21:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:21:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:21:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:21:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..90decbbbac8a373496e2226acd6bdf33340f1708 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/debug.log @@ -0,0 +1,353 @@ +[02-19 18:07:41|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 18:07:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:07:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:07:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 18:07:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 18:07:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 18:07:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 18:07:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 18:07:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 18:07:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 18:07:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 18:07:50|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 18:07:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 18:07:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 18:07:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 18:07:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 18:07:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 18:07:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 18:07:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 18:07:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 18:07:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 18:07:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 18:07:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 18:07:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 18:07:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 18:07:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 18:07:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 18:07:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 18:07:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 18:07:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 18:07:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 18:07:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 18:07:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 18:07:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 18:07:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 18:07:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:07:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:07:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:07:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 18:07:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 18:07:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:07:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:07:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:07:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 18:07:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:07:55|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 18:07:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:07:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 18:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 0.96 s +[02-19 18:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 18:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 18:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 18:07:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 18:07:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 18:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 18:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 18:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 18:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 18:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 18:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 18:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 18:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:07:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:08:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 18:08:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 18:08:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.03 s +[02-19 18:08:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:08:28|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:28|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:29|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 18:08:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:08:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/config.yaml +[02-19 18:08:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:08:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:08:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Toward the end of the video, the camera pans across the banquet hall, capturing the rows of green chairs and the tables set for an event. The video concludes with a final sweep of the room, highlighting the wooden ceiling beams and chandeliers, and ensuring that all spatial relationships and environmental cues remain consistent throughout the entire exploration.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 18:08:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_4.json +[02-19 18:08:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:08:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:08:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_3.mp4 +[02-19 18:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 18:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 18:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 18:08:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 18:21:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 18:21:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:21:46|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:21:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:21:47|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 18:21:48|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..5576a65c64275563a7caae74da1e125daf2484bd --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a sweeping view of the banquet hall, highlighting the wooden truss ceiling with its intricate beams and ornate chandeliers. The camera then moves to the right, revealing the buffet table adorned with silver chafing dishes and decorative lanterns, while also capturing the large windows and glass doors leading to the Pacific Terrace. The scene transitions smoothly as the camera begins to retrace its path, returning to the left side of the room near the glass doors, where the spatial consistency of the white pillars and potted plants is maintained.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..90edc72d7a412e8f9ed797078ecce381fddaa998 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60a691690176f37a8f417784a02e633b2fea52eaffe1abc827f2173a28286971 +size 2105614 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..1ab97e6375060b2b501fb7d3393d1a8687e45da9 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the camera continues its journey through the banquet hall, focusing on the detailed architecture of the ceiling and the evenly spaced chandeliers. The middle portion of the video revisits the buffet area, emphasizing the decorative elements and the spatial coherence of the environment. The camera then moves back toward the left side of the room, capturing the glass doors and the outdoor terrace, ensuring that the spatial relationships between objects remain consistent.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..496290d2f23ad822d25fd7ab19999aef3903efcd --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1585fe8c0d5f39aa93990a5c9afa7538f063f806ad175cbb4fdda4f98ddedc7c +size 1869670 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..bff875321313ca81058afc3613376f20e866d220 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"In the third segment, the camera tilts upward to showcase the ceiling's wooden beams and the intricate design of the chandeliers, providing a detailed view of the room's upper structure. The video then transitions to a broader perspective, capturing the entire banquet hall from a new angle, highlighting the rows of green chairs and the tables set for an event. The final segment returns to the buffet area, reinforcing the spatial consistency of the environment before concluding with a panoramic view of the entire room.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8c7f7f5f172e3cb125798b334a31a5b44f973ae3 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:96307c42c9d9cb33b1dfcacc3b61e803d01b2382995a46c6101fd65da546a1c3 +size 1702384 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..692eb882a8286a1adf779b433e419d6089fc202b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The fourth segment of the video focuses on the left side of the banquet hall, near the glass doors leading to the Pacific Terrace. The camera captures the white pillars and potted plants, ensuring that these elements remain consistent with their earlier appearances. The video then moves back toward the center of the room, revisiting the area around the pillars and chandeliers, and finally tilts upward to emphasize the ceiling's architectural details once more.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f57ad31eef1cd59e73e8108fcbb53d077a003f35 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4be29a21161c5b20d895042b1d09513114a59bc9f5cd9cac072453673417ddf5 +size 1787716 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..db88c1cfffb0b6975a2e3297238feaab064919a0 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Toward the end of the video, the camera pans across the banquet hall, capturing the rows of green chairs and the tables set for an event. The video concludes with a final sweep of the room, highlighting the wooden ceiling beams and chandeliers, and ensuring that all spatial relationships and environmental cues remain consistent throughout the entire exploration.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f4b824820a15f214bc6b151b6efeee893b089e48 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b41f5de3308181ceb9f4b3bdcfed1ff0c89855ade1c7c695aa5fce37ea8b7b8 +size 2001120 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36c939400425b7b628011608b9d87ef849b3ed7b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..14ab3765250b4dbd8b14d4336b5ed57e89e4e4f9 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/console.log @@ -0,0 +1,22 @@ +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 16:56:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 16:56:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:57:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/config.yaml +[02-19 16:57:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 16:57:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 16:57:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.json +[02-19 16:57:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 16:57:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 17:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:10:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:10:42|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..eaab3b5197f98407f30c8ddf06acdcd3953f8c61 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/debug.log @@ -0,0 +1,353 @@ +[02-19 16:56:27|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 16:56:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 16:56:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 16:56:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 16:56:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 16:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 16:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 16:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 16:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 16:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 16:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 16:56:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 16:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 16:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 16:56:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 16:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 16:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 16:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 16:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 16:56:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 16:56:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 16:56:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 16:56:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 16:56:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 16:56:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 16:56:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 16:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 16:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 16:56:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 16:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 16:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 16:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 16:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 16:56:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 16:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 16:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 16:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 16:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 16:56:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 16:56:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 16:56:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 16:56:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 16:56:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 16:56:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 16:56:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 16:56:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 16:56:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.03 s +[02-19 16:56:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 16:56:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 16:56:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 16:56:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.66 s +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.67 s +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.67 s +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 16:56:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:57:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 16:57:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 16:57:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.26 s +[02-19 16:57:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:57:16|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:16|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 16:57:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 16:57:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/config.yaml +[02-19 16:57:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 16:57:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 16:57:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='80-100%: In the final segment, the camera completes its near 360-degree loop, revisiting the original framing of the statue. It captures the full composition once more, ensuring that all spatial elements, including the plaza, benches, and background structures, remain visually consistent and coherent.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 16:57:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.json +[02-19 16:57:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 16:57:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 16:57:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4 +[02-19 16:57:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 16:57:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 16:57:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 16:57:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 17:10:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 17:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:10:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:10:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:10:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 17:10:42|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3c6ea1bb6269027e7a66251aebb8565f931dcee1 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"0-20%: The video opens with a frontal view of the bronze family statue, showcasing its detailed craftsmanship and the inscription on its base. The surrounding area features a circular brick plaza, wooden benches, and a prominent palm tree, with a modern building visible in the background.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..97a56d3482561a522f7029354f21c0da1cca053e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:207542dbfacba6b0f04ac3817d71841999adfda9faa1ffa189016fcf17e87d0a +size 1391438 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..4a1948c183a3ca8b016707b778364cc38632d5b1 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"20-40%: The camera begins a slow horizontal pan to the right, revealing the side profile of the statue and providing a view of the wooden benches and greenery encircling the plaza. The background remains consistent, with the building and palm tree staying in frame.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bf14e4cfc2d0c8861781b98c2bb6d15fdf98d028 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1e5903f3c424d8be6c14b0da5ab4a1e38d58750d774192a7db5db99ecb0e455f +size 1274078 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..6ac937b02cc46d07b2f59c25da97b23774fd1b5c --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"40-60%: Continuing the pan, the camera moves further around the statue, capturing its rear view and emphasizing the flowing drapery of the figures' garments. The plaza's circular design and the evenly spaced benches become more apparent as the camera explores different perspectives.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9a4681bdd9d3c6d41aaa2622f6546cf5d1daafea --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5d3c418bd84877bf26c3d87e8a899f68c7544b08aba03b5cd9b9dd7b5710904 +size 1541157 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..d22064c5a19dae9f9985d9c6314e8c8938e6c5b4 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"60-80%: The camera starts to retrace its path, moving back toward the initial frontal view of the statue. It zooms in slightly to focus on the faces and hands of the figures, highlighting their emotional expressions before pulling back to a wider shot.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b66eb634cb9e9cdb2b71a7856d2dbe3dbf63dd5d --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9d31f0b17b8d5f20b7eeb4839e6462004ba2d0bdc8563ca4c7e83ebdeefc6aa +size 1630647 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..057239be93ef526d98b653dd414bf492dd6e5c93 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"80-100%: In the final segment, the camera completes its near 360-degree loop, revisiting the original framing of the statue. It captures the full composition once more, ensuring that all spatial elements, including the plaza, benches, and background structures, remain visually consistent and coherent.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..848793f81b513e9e4872f2694dd9591012418f5e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fef300b475b35cd6c427b20f660759583d4a074c0f72925c0cc4b2a04f4b78fe +size 2775555 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08db1dc00f4b35d2f294b5d157b19bfc70db4f4c --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/console.log new file mode 100644 index 0000000000000000000000000000000000000000..57caee78cf091e59de455e574dfed95bf86e2b7c --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/console.log @@ -0,0 +1,22 @@ +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 16:56:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 16:56:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/config.yaml +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_4.json +[02-19 16:57:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 17:10:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:10:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:10:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:10:26|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6909ca6c8c4ae11344e49ec810fe47c4ddb485e4 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/debug.log @@ -0,0 +1,353 @@ +[02-19 16:56:17|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 16:56:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 16:56:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 16:56:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 16:56:26|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 16:56:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 16:56:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 16:56:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 16:56:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 16:56:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 16:56:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 16:56:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 16:56:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 16:56:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 16:56:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 16:56:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 16:56:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 16:56:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 16:56:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 16:56:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 16:56:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 16:56:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 16:56:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 16:56:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 16:56:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 16:56:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 16:56:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 16:56:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 16:56:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 16:56:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 16:56:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 16:56:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 16:56:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 16:56:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 16:56:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 16:56:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 16:56:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 16:56:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 16:56:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 16:56:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 16:56:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 16:56:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 16:56:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 16:56:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.02 s +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.68 s +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.68 s +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.68 s +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 16:56:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 16:56:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 16:56:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 16:56:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.29 s +[02-19 16:56:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:57:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:57:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 16:57:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/config.yaml +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 16:57:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="The final segment of the video provides a comprehensive overview of the auditorium, starting with a detailed look at the side corridor and the additional seating sections. The camera then returns to the main stage area, focusing on the wooden podium and the projector screen, while capturing the ceiling's lighting fixtures from various angles. The video concludes by moving through the central seating area one last time, ensuring that all spatial elements are consistently represented, thus providing a thorough and coherent exploration of the auditorium." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_4.json +[02-19 16:57:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_3.mp4 +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 16:57:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 16:57:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 16:57:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 17:10:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 17:10:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:10:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:10:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:10:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 17:10:26|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..4e12f91ad7abc8984d4f29967fac476c2630c02f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a sweeping view of the auditorium's central seating area, showcasing rows of patterned beige seats arranged in a semi-circular fashion. The camera then moves to the front, capturing the wooden podium with black screens and the large projector screen above it, while also highlighting the curved ceiling with its track lighting. This initial segment sets the stage for the detailed exploration of the auditorium's key features.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2b8118bc1b63242d2ea14fcce8146fe309654a69 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:64766cc7e1fb7b4a51c7e8a7c85e548ddeda5e3f791fbed3bb6ded2a8321943e +size 1262281 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..bd665ed79c4e6b705181a8c7bf51cc544d599e07 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the camera transitions into the side corridor, revealing a series of doors and additional seating sections. The movement through this corridor provides a sense of depth and perspective, showcasing the spatial continuity of the auditorium. The camera then returns to the main stage area, focusing on the podium and projector screen from various angles, while also capturing the ceiling's intricate lighting design.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6631261d9e00db19b6c52c6e71424b8c4f174660 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e0ce3c3d9a9ba2c72d60dbe2bf8f71c37e9f650a7227f6f412d0e7d53cf76c74 +size 1285051 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..4585e8d07a8c0a3dcb90c5e0853318d6b540f449 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"In the middle portion of the video, the camera explores the upper levels of the auditorium, offering a broader view of the seating arrangement and the overall spatial layout. The focus shifts to the ceiling, where the track lighting and spotlights are prominently featured, emphasizing the curved design and the distribution of light throughout the space. The camera then moves back to the central seating area, ensuring that all spatial elements are consistently represented.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1436a6a939c2782e3fa0743126218e1f4d3a131b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:031dbcaa65b68f7287fe2c402e61baeeee51752ddc120238be24520632619f82 +size 1158779 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..0c2dffa782672a8875b9abee3bea667e3d5b85cb --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Toward the end of the video, the camera revisits the central seating area from different perspectives, reinforcing the uniformity of the patterned seats and the symmetrical arrangement. The focus then shifts to the stage area, where the wooden podium and projector screen are highlighted once more, with attention given to the ceiling's lighting fixtures. The video concludes by moving through the side corridor again, ensuring that all previously seen elements are accurately represented, thus maintaining spatial coherence throughout the entire auditorium.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..93909a1988fef1d334fd909481ac5f3453eddff2 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e5fb6f0dc71c9a4a48e741a28efa601da4a42020f7cf15c9699cf8e68b87aad1 +size 951672 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..05cbfa2d1940eea0f145c3e7e3a40a70033245c9 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The final segment of the video provides a comprehensive overview of the auditorium, starting with a detailed look at the side corridor and the additional seating sections. The camera then returns to the main stage area, focusing on the wooden podium and the projector screen, while capturing the ceiling's lighting fixtures from various angles. The video concludes by moving through the central seating area one last time, ensuring that all spatial elements are consistently represented, thus providing a thorough and coherent exploration of the auditorium.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..68bbb0eb6e122e5eacd1e987ab6abab89fd11a74 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1299dea094bb0f8ff0c285c7d79cbcc7c3ea9c4cf4c7e06b77247ca725cf6462 +size 1416559 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34ba920c6af97512f3f68f643c7f5bcb29b3ddef --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/console.log new file mode 100644 index 0000000000000000000000000000000000000000..5bf6f1a2ad1c7b0fa780b7b56e58080cf98da476 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/console.log @@ -0,0 +1,22 @@ +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:08:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/config.yaml +[02-19 18:08:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:08:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:08:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_4.json +[02-19 18:08:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:08:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:21:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:21:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:21:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:21:36|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..495d87c62cd6ca868d74f49caa6fcbb79aeff1e6 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/debug.log @@ -0,0 +1,353 @@ +[02-19 18:07:30|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 18:07:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:07:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:07:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 18:07:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 18:07:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 18:07:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 18:07:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 18:07:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 18:07:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 18:07:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 18:07:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 18:07:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 18:07:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 18:07:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 18:07:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 18:07:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 18:07:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 18:07:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 18:07:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 18:07:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 18:07:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 18:07:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 18:07:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 18:07:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 18:07:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 18:07:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 18:07:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 18:07:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 18:07:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 18:07:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 18:07:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 18:07:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 18:07:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 18:07:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 18:07:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:07:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:07:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 18:07:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:07:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 18:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:07:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 18:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:07:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:07:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 18:07:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:07:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.17 s +[02-19 18:07:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 18:07:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 18:07:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 18:07:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:45|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:07:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:07:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:07:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:08:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 18:08:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 18:08:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.70 s +[02-19 18:08:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:08:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:08:17|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 18:08:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:08:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/config.yaml +[02-19 18:08:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:08:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:08:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="80-100%: Toward the end of the video, the camera begins to retrace its path once more, moving back toward the starting point. It revisits the initial section of the playground, showing the wooden play structure, slides, and blue spring rider toy again. The camera then completes its loop, returning to the original viewpoint. Throughout this final segment, the spatial consistency of the playground is emphasized, with all objects and structures appearing in their expected locations, reinforcing the video's focus on spatial and environmental coherence." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 18:08:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_4.json +[02-19 18:08:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:08:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:08:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_3.mp4 +[02-19 18:08:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 18:08:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 18:08:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 18:08:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 18:21:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 18:21:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:21:34|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:21:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:21:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 18:21:36|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..f5db875222b4f9a98bf738ecd3c12bc2211173b4 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"0-20%: The video opens with a close-up view of a section of the playground, focusing on a wooden play structure with two metal slides, a blue spring rider toy on the sandy ground, and a concrete bench nearby. The background includes a red wooden fence, residential houses, and tall evergreen trees. The camera begins a slow pan to the left, revealing a pathway bordered by rocks and more of the playground's perimeter.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eb8d310ae1e12bacdb469e8e14666e772b3d9a0 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed12a7eef0fdeae6a3d9cd0a506bb875a0661df46aa7888f725a232dcf3f84ad +size 1980032 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..ed4aa644cbf09b0f8311838ad55b1a6a313f5228 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"20-40%: As the camera continues its leftward pan, it showcases additional playground elements, including more concrete benches and a different section of the wooden play structure with slides and climbing features. The background remains consistent with residential houses and trees, but new details like a small shed or pavilion become visible in the distance. The camera movement is smooth, maintaining a steady pace as it explores the playground's layout.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7a51c98c76a67522da82d2eb64beb82bca9725fa --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de04bfe71249e4f28879c0e1ec4bea64ad340c2ac2f318d54498a9030e07dde1 +size 1745468 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..a0d8be7c559688a1c21e94f5deda8af9adb5385c --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"40-60%: The camera completes its leftward pan and begins moving rightward, retracing part of its path. It revisits the initial section of the playground, showing the same wooden play structure, slides, and blue spring rider toy. The camera then continues further right, revealing more of the playground's expanse, including additional benches, swings, and a larger open area with grass and trees in the background. The lighting remains consistent, indicating a single time of day.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b463282e78b545836bb5ca57ae78c5c54facac7d --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c7ef9c6521492a5aad448e692f62c033727ed9ed5f8c052c3ae2fefc81c90349 +size 1758541 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..41ae27e9cf63881715f2d6617bc6ad5b1862bf0e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"60-80%: The camera continues its rightward pan, exploring the far side of the playground. It captures a wide view of the area, including a large grassy field, more benches, and a pathway leading away from the playground. The background now includes a road with passing vehicles, including a white truck with 'Atlas' branding. The camera movement is deliberate, ensuring that all key elements of the playground are captured from this new angle.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a261d5bfb215fe2edab4ec4824f9e58356e31aae --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:532403ec5d97e305196b04dc547c0fd8d463a36ecc032f4417639cdc3de3c50f +size 1580274 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..05d0f0c5bd7bdcfae332583d4336e6c43e075c90 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"80-100%: Toward the end of the video, the camera begins to retrace its path once more, moving back toward the starting point. It revisits the initial section of the playground, showing the wooden play structure, slides, and blue spring rider toy again. The camera then completes its loop, returning to the original viewpoint. Throughout this final segment, the spatial consistency of the playground is emphasized, with all objects and structures appearing in their expected locations, reinforcing the video's focus on spatial and environmental coherence.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7f6e12bc185b2b0c444e5e601392c35a2e0d9993 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb5b452acd73eac9ec0e169d7a0bfbf4616cffba8687eb42d073730d96f91004 +size 1093842 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1c05db5509bea5d677352f1c4017a1f4d6fa870 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/console.log new file mode 100644 index 0000000000000000000000000000000000000000..b1a1c5dd21f0d396890d94e2b582441f5ede4ecc --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/console.log @@ -0,0 +1,22 @@ +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:34:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:34:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/config.yaml +[02-19 19:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 19:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 19:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_4.json +[02-19 19:34:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 19:49:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 19:49:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 19:49:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 19:49:18|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e63b9eda6c351dd4b5c623ebf7651ca430c0bca8 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/debug.log @@ -0,0 +1,353 @@ +[02-19 19:34:01|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 19:34:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:34:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:34:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 19:34:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 19:34:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 19:34:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 19:34:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 19:34:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 19:34:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 19:34:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 19:34:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 19:34:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 19:34:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 19:34:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 19:34:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 19:34:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 19:34:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 19:34:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 19:34:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 19:34:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 19:34:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 19:34:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 19:34:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 19:34:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 19:34:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 19:34:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 19:34:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 19:34:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 19:34:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 19:34:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 19:34:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 19:34:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 19:34:14|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 19:34:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 19:34:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:34:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:34:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 19:34:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:34:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:34:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:34:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 19:34:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:34:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 19:34:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:34:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:34:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 19:34:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:34:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:34:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 19:34:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 19:34:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.47 s +[02-19 19:34:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:34:50|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:50|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:34:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 19:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/config.yaml +[02-19 19:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 19:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 19:34:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='80-100%: Toward the end, the video provides a final overview of the church interior, combining wide shots of the main nave with close-ups of key features like the altar and ceiling frescoes. The camera retraces its path, ensuring that all previously seen elements are depicted consistently. The video concludes with a sweeping view of the entire church, leaving the viewer with a lasting impression of its grandeur and spiritual atmosphere.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 19:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_4.json +[02-19 19:34:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 19:34:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_3.mp4 +[02-19 19:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 19:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 19:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 19:34:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 19:49:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 19:49:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 19:49:16|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 19:49:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 19:49:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 19:49:18|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..ae7594df37924e8266b210595ec0da78484c039c --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"0-20%: The video begins with a panoramic view of the church's main nave, capturing the rows of wooden pews and the ornate altar at the far end. The camera slowly pans across the space, highlighting the arched side walls and doorways adorned with religious paintings and statues. The initial focus is on establishing the overall layout and grandeur of the church interior.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4e09f065018234078078e30a2d2d1b6165b8d67e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f459c45cb0d0772de7893d12d0f0460a3acf1815c21d0c91f2db8d87a24fe6d2 +size 1014355 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..ced1f57dc6c8eaac48df1cd8545c3eb038abd40a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"20-40%: The camera then tilts upward to showcase the intricately painted ceiling frescoes and the elegant chandeliers hanging from the ceiling. This segment emphasizes the vertical perspective and the detailed artwork above. The camera also captures the side altars and balconies, providing a sense of the church's depth and complexity.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3849108518340555d660fe2000ee3361dc3d7c4c --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:19c17a9867c177cf66cc0a68519d6e9ed254f227daf2c2c3a402002828fe4476 +size 2192341 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..d84439997d5735c3287500371f74336eca2767eb --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"40-60%: Moving forward, the video focuses on the altar area, offering close-up views of the golden decorations, religious icons, and candles. The camera explores the altar from different angles, highlighting the craftsmanship and spiritual significance of the space. The segment also includes a brief return to the main nave, reinforcing the spatial continuity.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..80810722c4d4bc6c76ecee2d17995ae0e4c0f97a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5575b6c3948073d031e3934ec4e1f944ae64a0329c894feaefb1d676fb5bd585 +size 2301935 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..b391b68123880057660a7225802459e8c8dbda1b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"60-80%: The exploration continues with a detailed examination of the arched side walls and doorways, capturing the symmetry and architectural beauty of the church. The camera pans along these walls, revealing additional religious paintings and statues. This segment also includes a revisit to the altar area, ensuring that the viewer has a comprehensive understanding of the space.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..21f8747cf30df57f3d5d59d6d846f79ee95a2b4f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de5bbd139041843f91371a6bfa8cade0d506113eed6d29b5c43427b421e80daa +size 1789853 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..183d432403bdc8149999abbf8b766be9711868ef --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"80-100%: Toward the end, the video provides a final overview of the church interior, combining wide shots of the main nave with close-ups of key features like the altar and ceiling frescoes. The camera retraces its path, ensuring that all previously seen elements are depicted consistently. The video concludes with a sweeping view of the entire church, leaving the viewer with a lasting impression of its grandeur and spiritual atmosphere.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8046dc2eb1df13d18b2018e7f53f9fffd85d88af --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbab37a7932119af89ce6436f00b6cbd7ca0bf200f4d58d4c284c046694ee564 +size 1749116 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbb7556c2e3dee68da965b8a9605e23095ea7a22 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/console.log new file mode 100644 index 0000000000000000000000000000000000000000..c96093f7b06879244a54699cdf295dad8c11b9a0 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/console.log @@ -0,0 +1,22 @@ +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/config.yaml +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.json +[02-19 16:56:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 17:10:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:10:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:10:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:10:13|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e9ff7d48cf0d2c75a33b20bc663bfccabcaad1dd --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/debug.log @@ -0,0 +1,353 @@ +[02-19 16:56:03|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 16:56:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 16:56:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 16:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 16:56:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 16:56:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 16:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 16:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 16:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 16:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 16:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 16:56:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 16:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 16:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 16:56:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 16:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 16:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 16:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 16:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 16:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 16:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 16:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 16:56:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 16:56:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 16:56:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 16:56:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 16:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 16:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 16:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 16:56:16|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 16:56:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.04 s +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.65 s +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.65 s +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 16:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 16:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.05 s +[02-19 16:56:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:51|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 16:56:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/config.yaml +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 16:56:52|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="The video concludes with the camera returning to the front-left view of the loader, mirroring the initial shot. The final scenes include a few more zoom-ins on the bucket and wheels, reinforcing the detailed examination of the loader's features. The camera then pulls back to a wide shot, showing the entire machine in the context of the dry field, with the dead tree stump and trees in the background. The video ends with a steady, comprehensive view of the loader, ensuring all spatial and environmental elements are consistently represented." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.json +[02-19 16:56:52|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4 +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 16:56:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 16:56:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 16:56:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 17:10:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 17:10:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:10:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:10:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:10:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 17:10:13|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..09d89704b09f9a159d1ae2533d8ce3563875aa2b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up of the front-left side of the yellow Caterpillar 950F wheel loader, showcasing its large bucket and rugged tires. The camera begins a smooth horizontal movement, starting to circle the loader while maintaining a consistent distance. The initial focus is on the front, gradually transitioning to the side as the camera orbits. The background features dry grass, green trees, and a dead tree stump on the right, which become part of the frame as the camera moves.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..24d79ce5e6c0ff17a4a8d6df0d109f078ab8ef44 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee889852e3422d905bcd2727dd4fd4d91cf54a2716e49143d64714816662268 +size 1241984 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..4abae1114c54a90898006dfdc7ad78b2bf268fa3 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues its orbit, it shifts to a higher vertical perspective, capturing the top of the loader's cab and then tilting down to focus on the large rear tires and the exhaust stack. The zooming in and out becomes more noticeable, with the camera moving closer to highlight details like the bucket's teeth and the tire treads. The background elements, including the green trees and distant hills, remain visible, entering and exiting the frame as the camera explores different angles.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0cdacdf3decb161ee490160c14eaee34a3502138 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fa0aaeea7659cf9ec0d7da132844fbf9b71daa76f6b75de57b3da28a05df51a7 +size 1712299 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..2fb5e487ceba394d44c2eeb71f36e78d1d9bcdf1 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera then moves to the rear of the loader, providing a clear view of the side decals and mechanical components. It tilts up to show the top of the cab and then down to emphasize the large rear tires and the exhaust stack. The zooming in and out continues, with close-ups of the loader's side features and the bucket. The background remains consistent, with the green trees and distant hills providing a stable environment as the camera re-enters familiar angles and perspectives.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..32af2dec8d731b3cd07f193652d4b56d7b4f403a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0f86b834539a2e630f46eee3b5ef5dc0b59129aff6fe15f8a32f932ffdbc239 +size 2150702 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..9ce4b843992b9eb0937c7a0eb82cbc154675eb50 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the next segment, the camera completes its orbit by moving back towards the front of the loader, now focusing on the front-right side. The vertical perspective shifts again, with the camera tilting up to capture the top of the cab and then down to emphasize the front tires and the bucket. The zooming in and out becomes more pronounced, with close-ups of the bucket's teeth and the tire treads. The background remains consistent, with the green trees and distant hills providing a stable environment as the camera re-enters familiar angles and perspectives.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cec709e79d787a382c7ed5de17c074d61dc05880 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:381de3d268503507fafc2258662bae117a533d7f8210e82a3836d7c0390f396b +size 1792018 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..1610c231a79879954fff92f04495e884947dd76b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with the camera returning to the front-left view of the loader, mirroring the initial shot. The final scenes include a few more zoom-ins on the bucket and wheels, reinforcing the detailed examination of the loader's features. The camera then pulls back to a wide shot, showing the entire machine in the context of the dry field, with the dead tree stump and trees in the background. The video ends with a steady, comprehensive view of the loader, ensuring all spatial and environmental elements are consistently represented.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3be1e38a594c51f893b263de2997cbfc8f276f42 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:011d882fbdf4986607d274b94d7d519df53c96d9a1629fa8c8cda33667d828de +size 2427925 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f7d3a79eb297eebbe78769cb70f0d6880464d97 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/console.log new file mode 100644 index 0000000000000000000000000000000000000000..d8524f9738a8844395181d6170a099511c825f0c --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/console.log @@ -0,0 +1,22 @@ +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/config.yaml +[02-19 16:56:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 16:56:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 16:56:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_4.json +[02-19 16:56:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 16:56:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 17:10:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:10:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:10:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:10:14|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1564babfdc0cd35a2403f69ebfdbab8bccfcc3b0 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/debug.log @@ -0,0 +1,353 @@ +[02-19 16:56:06|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 16:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 16:56:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 16:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 16:56:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 16:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 16:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 16:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 16:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 16:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 16:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 16:56:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 16:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 16:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 16:56:15|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 16:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 16:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 16:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 16:56:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 16:56:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 16:56:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 16:56:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 16:56:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 16:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 16:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 16:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 16:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 16:56:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 16:56:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 16:56:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 16:56:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 16:56:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:20|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 16:56:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 16:56:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 16:56:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.22 s +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 16:56:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.65 s +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.66 s +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.66 s +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 16:56:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 16:56:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.67 s +[02-19 16:56:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 16:56:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 16:56:54|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 16:56:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 16:56:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/config.yaml +[02-19 16:56:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 16:56:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 16:56:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="Toward the end, the camera completes its circular journey around the park restroom building, revisiting the left side and open front area. The beige siding, dark roof, and lattice fence are once again in view, along with the picnic tables and tree in the foreground. The camera captures the building's details, such as the skylights, brown wooden overhang, and closed garage doors, from multiple angles. The background elements, including the green lawn, trees, and neighboring houses, reappear as the camera returns to the starting point. The video concludes with a final shot of the restroom building, emphasizing its architectural features and the surrounding park environment." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 16:56:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_4.json +[02-19 16:56:54|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 16:56:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 16:56:54|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_3.mp4 +[02-19 16:56:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 16:56:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 16:56:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 16:56:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 17:10:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 17:10:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:10:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:10:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:10:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 17:10:14|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..2b194e3342dbef5826705b074865447be3e337b4 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up view of the left side of the park restroom building, highlighting its beige siding, dark roof, and a small window with green shutters. A concrete pathway leads to the building, and a tree is visible in the foreground. The camera begins its pan to the right, capturing the open front area with picnic tables and a lattice fence on the side. The initial segment sets the stage for the circular exploration of the building, establishing key spatial anchors like the picnic tables, tree, and pathway.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b08b4ffc8340e6214bd1ab269b13c10680f4fd31 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be23127b43374ea2dc9cbfb1104f7f7f198281e70493c3b859edc2354473d21c +size 1209971 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..5de885060b9f997946ff9420605864a87b533919 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues its pan, it moves along the right side of the restroom building, revealing more details such as the closed garage doors, a small door near the lattice fence, and a brown wooden overhang above the entrance. The camera tilts slightly upward to capture the roofline and skylights, then moves downward to show the base of the building and the concrete pathway. The middle segment emphasizes the building's architectural features and maintains a consistent vertical perspective, while background elements like green lawns, trees, and neighboring houses begin to reappear as the camera loops back toward the starting point.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2ba5d4f9cd0fbaa5b5ebb02a15a2bf7825d58154 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8967ec9bd170e2c8043e402ecfbe7286eefb158f5305ef7c46c325dcfaa94786 +size 1787849 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..a28e7c77048f696250407aa9282a128a74f66e10 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"In the third segment, the camera focuses on the rear and left sides of the restroom building, capturing the lattice fence, closed garage doors, and small door near the fence. The camera continues its circular motion, maintaining a consistent scale and perspective, while background elements like green lawns, trees, and neighboring houses re-enter the frame. The red bicycle rack becomes visible as the camera approaches the front of the building, and the scene begins to loop back toward the initial viewpoint.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4d1dd52300ca57d43c34e7b1f8cc3819c9290822 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be169c221373c77b8dc4cad6988f15a8d6a6c96b10bfd412b95d6e75eb726beb +size 1608047 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..89ccc371559f717e97eeb7a9fc3295bdd4dac2ee --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The fourth segment shows the camera completing its pan around the restroom building, revisiting the left side and open front area. The beige siding, dark roof, and lattice fence are once again in view, along with the picnic tables and tree in the foreground. The camera captures the building's details, such as the skylights, brown wooden overhang, and closed garage doors, from multiple angles. The background elements, including the green lawn, trees, and neighboring houses, reappear as the camera returns to the starting point.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d443acdcb468cde3929bdc4af40a4769f3f43f50 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4c16d84ac8afedf80f0e55abbc212eb1380a8fb39b975cead36dacac06113ed +size 1796173 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..6a0fa9d7d97fc693aecb889fda6375f807819555 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Toward the end, the camera completes its circular journey around the park restroom building, revisiting the left side and open front area. The beige siding, dark roof, and lattice fence are once again in view, along with the picnic tables and tree in the foreground. The camera captures the building's details, such as the skylights, brown wooden overhang, and closed garage doors, from multiple angles. The background elements, including the green lawn, trees, and neighboring houses, reappear as the camera returns to the starting point. The video concludes with a final shot of the restroom building, emphasizing its architectural features and the surrounding park environment.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d4879e82afbd75090c52d6b830b7f1057721cb99 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8171ac891fd3e3d2866c0b55b72b970796590756f80cfd9013bab8af5edc7b5b +size 2444326 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a22bbdfec94a6e6f8bb8714f60ee08d04fb11eb --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..6287bc5df809a39885677f9f25575c968e226e03 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/console.log @@ -0,0 +1,22 @@ +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 17:00:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 17:00:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 17:01:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/config.yaml +[02-19 17:01:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 17:01:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 17:01:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.json +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 17:01:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 17:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:15:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:15:35|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..09a74d4d2acd2af01e2ff216fa65f0242859f2ea --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/debug.log @@ -0,0 +1,353 @@ +[02-19 17:00:23|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 17:00:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 17:00:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 17:00:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 17:00:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 17:00:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 17:00:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 17:00:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 17:00:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 17:00:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 17:00:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 17:00:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 17:00:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 17:00:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 17:00:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 17:00:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 17:00:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 17:00:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 17:00:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 17:00:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 17:00:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 17:00:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 17:00:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 17:00:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 17:00:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 17:00:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 17:00:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 17:00:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 17:00:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 17:00:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 17:00:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 17:00:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 17:00:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 17:00:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 17:00:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 17:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 17:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 17:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 17:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 17:00:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 17:00:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 17:00:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 17:00:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 17:00:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 17:00:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 17:00:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 17:00:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 17:00:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 17:00:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 17:00:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-19 17:00:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 17:00:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 17:00:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 17:00:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:00:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 17:00:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 17:00:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 17:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 17:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 17:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.89 s +[02-19 17:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 17:01:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:01:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 17:01:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 17:01:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/config.yaml +[02-19 17:01:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 17:01:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 17:01:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='Toward the end, the camera continues its horizontal pan, revisiting earlier viewpoints to complete the loop. The focus returns to the base of the monument, with the camera tilting downward to capture the detailed stonework once again. The background elements, such as the trees and the modern building, remain consistently visible, providing a sense of continuity. The video concludes with a final view of the monument from a slightly elevated angle, emphasizing its grandeur and the surrounding park environment.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 17:01:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.json +[02-19 17:01:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 17:01:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 17:01:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4 +[02-19 17:01:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 17:01:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 17:01:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 17:01:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 17:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 17:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:15:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:15:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:15:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 17:15:35|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..c75548cae09a8f3d9e445c1e80a23c71f53734b6 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up view of the monument's base, highlighting the intricate stonework and the paved path surrounding it. The camera then begins a slow upward tilt, gradually revealing the seated bronze statue within the arched structure. During this initial phase, the background includes tall trees and a modern angular building to the left, which remain partially visible.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..252b7a140847081106eeb2f92c73a4edf187c3fc --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8c8368a9295bc55923fe5194dc260d5aeb8e17f16f26894db9b3696143d8914 +size 925271 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..ff3eea5b2fa52cc5f1913506f652f2274f80826f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues its movement, it starts a horizontal pan around the monument, capturing it from various angles. The pan is smooth and systematic, ensuring that the monument remains the central focus while the background elements, such as the trees and the modern building, stay consistently framed. The camera maintains a steady pace, allowing viewers to appreciate the monument's details from multiple perspectives.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c4a66fe1a9905a9679ec9d41536a66fbaaee1cf8 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36ffddd0846643176e1ec18ba1bc9234de5b702c64c5426e497b2d5ed6594609 +size 1031858 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..0a5dd4b4f2fd3c68dd16e9967c555f3a9595ba3e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"During the middle portion of the video, the camera continues its horizontal pan, now focusing more on the middle section of the monument. The seated bronze statue is prominently featured, and the camera occasionally tilts to provide closer views of the inscriptions on the base. The background remains stable, with the trees and modern building consistently visible as the camera moves around the monument.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9776aa49dd7d9d11f519e55ebec40d70f979dd4d --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6369f094b1007a3ede028122273f93cac04f074c0bf125f91d1ad97375d02b80 +size 1289704 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..996725c358786be79a8e062831c6bc656e1bd693 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the latter part of the video, the camera begins to tilt upward, focusing on the upper portion of the monument. This shift in perspective reveals the statues at the apex, adding a new layer of detail to the scene. The background elements, including the trees and the modern building, re-enter the frame as the camera completes its loop, maintaining spatial coherence. The lighting and positioning of these elements are preserved, ensuring a seamless transition between different angles.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9077d76caae3160ad2c768e04ae75dd6fa803873 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a292e424951e1ba2e0d46a11715cf94e2ed36a5a7c76fb8a3fba95e69bbdcedd +size 2005058 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..8acbcf7316111c95d34b94fec461a7677e385bb3 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"Toward the end, the camera continues its horizontal pan, revisiting earlier viewpoints to complete the loop. The focus returns to the base of the monument, with the camera tilting downward to capture the detailed stonework once again. The background elements, such as the trees and the modern building, remain consistently visible, providing a sense of continuity. The video concludes with a final view of the monument from a slightly elevated angle, emphasizing its grandeur and the surrounding park environment.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1671c8c8af01cb24b5b549d78cee6f73d9899775 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b57f1ce096f2d8f65d0ac2cf3f4ecb039de892b5ebb40a31fa3f77981913bf11 +size 2075053 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b17806607c14b9448a88fa84e099e6af1405a1df --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/console.log new file mode 100644 index 0000000000000000000000000000000000000000..e22eead9cf688b65299c8c64b84c9e680cb58bd5 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/console.log @@ -0,0 +1,22 @@ +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 17:11:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 17:11:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 17:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 17:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 17:11:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/config.yaml +[02-19 17:11:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 17:11:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 17:11:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.json +[02-19 17:11:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 17:11:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 17:25:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:25:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:25:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:25:05|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2e28f4f700125eff309ebf8b940e1dba82be3e3e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/debug.log @@ -0,0 +1,352 @@ +[02-19 17:10:52|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 17:11:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 17:11:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 17:11:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 17:11:02|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 17:11:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 17:11:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 17:11:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 17:11:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 17:11:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 17:11:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 17:11:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 17:11:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 17:11:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 17:11:02|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 17:11:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 17:11:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 17:11:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 17:11:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 17:11:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 17:11:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 17:11:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 17:11:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 17:11:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 17:11:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 17:11:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 17:11:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 17:11:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 17:11:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 17:11:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 17:11:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 17:11:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 17:11:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 17:11:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 17:11:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 17:11:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 17:11:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 17:11:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 17:11:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 17:11:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 17:11:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 17:11:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 17:11:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 17:11:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 17:11:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 17:11:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 17:11:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 17:11:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 17:11:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.02 s +[02-19 17:11:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 17:11:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 17:11:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 17:11:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 17:11:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.69 s +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.69 s +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.69 s +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 17:11:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 17:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 17:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 17:11:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 17:11:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 17:11:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 17:11:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.10 s +[02-19 17:11:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 17:11:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 17:11:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 17:11:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 17:11:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/config.yaml +[02-19 17:11:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 17:11:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 17:11:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a horizontal pan across the conference hall, revealing the coffered ceiling, fireplace, and arched windows. The camera then tilts downward to focus on the carpet and wall sconces, emphasizing the room's detailed textures. In the next segment, the camera continues to explore the space, moving back toward the fireplace and S-shaped table, ensuring spatial consistency. The following segment retraces the path, revisiting the stage area and long tables by the windows, while the camera maintains a smooth motion. Finally, the video concludes with a return to earlier seen areas, ensuring all spatial elements remain coherent and consistent." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 17:11:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.json +[02-19 17:11:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 17:11:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 17:11:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 17:11:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 17:11:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 17:11:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 17:25:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 17:25:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 17:25:03|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 17:25:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 17:25:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 17:25:05|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..14f30184c80c8e5a7d30e1d7ab942a1d38297b46 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a horizontal pan across the conference hall, revealing the coffered ceiling, fireplace, and arched windows. The camera then tilts downward to focus on the carpet and wall sconces, emphasizing the room's detailed textures. In the next segment, the camera continues to explore the space, moving back toward the fireplace and S-shaped table, ensuring spatial consistency. The following segment retraces the path, revisiting the stage area and long tables by the windows, while the camera maintains a smooth motion. Finally, the video concludes with a return to earlier seen areas, ensuring all spatial elements remain coherent and consistent.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6ad9a34139cf957e69ac0cd1163f8dbe32a56327 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7abae1ff350f0d7b9d4c5c05bf54262d5655515883a0b3c44da179f48d22940c +size 1058407 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c394a4a0636f5a8b41e2ff92cdbd286a6bee311 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/console.log new file mode 100644 index 0000000000000000000000000000000000000000..266999510cb839f3d562aa6d1cebf01c4cc596da --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/console.log @@ -0,0 +1,22 @@ +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:19:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:19:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:20:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/config.yaml +[02-19 19:20:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 19:20:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 19:20:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_4.json +[02-19 19:20:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:20:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 19:33:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 19:33:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 19:33:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 19:33:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c803cf670356a0a265db8846c547a3fbaa62925f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/debug.log @@ -0,0 +1,353 @@ +[02-19 19:19:30|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 19:19:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:19:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:19:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 19:19:39|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 19:19:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 19:19:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 19:19:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 19:19:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 19:19:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 19:19:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 19:19:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 19:19:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 19:19:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 19:19:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 19:19:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 19:19:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 19:19:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 19:19:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 19:19:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 19:19:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 19:19:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 19:19:40|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 19:19:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 19:19:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 19:19:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 19:19:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 19:19:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 19:19:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 19:19:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 19:19:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 19:19:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 19:19:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 19:19:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 19:19:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 19:19:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:19:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:19:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 19:19:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:19:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:19:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 19:19:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:19:45|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 19:19:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:19:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:46|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 19:19:46|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:19:47|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 19:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 19:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.61 s +[02-19 19:20:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:20:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:20:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 19:20:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:20:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/config.yaml +[02-19 19:20:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 19:20:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 19:20:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="80-100%: In the final segment, the camera continues to explore the locomotive from various angles, focusing on the front and side views. It captures the intricate details of the locomotive's structure, including the wheels, undercarriage, and safety features. The video concludes with a wide shot of the locomotive, emphasizing its position in the railway yard and the surrounding environment, maintaining spatial consistency and coherence throughout." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 19:20:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_4.json +[02-19 19:20:19|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:20:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 19:20:19|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_3.mp4 +[02-19 19:20:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 19:20:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 19:20:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 19:20:20|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 19:33:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 19:33:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 19:33:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 19:33:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 19:33:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 19:33:39|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3e81fd272c6240f80c8d174afc5137ab79621594 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"0-20%: The video opens with a close-up view of the front of the green Western Pacific locomotive, number 713. The camera slowly pans to the left, revealing the orange safety railings and steps leading up to the locomotive's cabin. The 'WESTERN PACIFIC' lettering is prominently displayed on the side. The background shows a hilly green landscape under a clear blue sky, with a blue locomotive visible in the distance.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ed33e72b0fd6abd4b8e2ea960fc027b99f1c6718 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9356b6766881905aa5fc0d3aa237695c49ea8ba822b9afd04177cd39240e23e +size 1351495 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..eccf3f53f3a3dbdf4243fa7da8b1103b89df9c79 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"20-40%: As the camera continues its circular movement, it tilts upwards to capture the roof of the locomotive, showcasing its vents and other features. It then tilts downwards to focus on the wheels and undercarriage, highlighting the mechanical details. The camera moves along the side of the locomotive, providing a full view of its length and the surrounding railway yard, which includes various equipment and another blue locomotive in the background.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7852d5df2a3534451d1cdc4ef070a8f54c608ade --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33fa95839d43c3e5f1a5a7bb1bcf7861cef2dee99e84f77e489be83c0d0ca73d +size 1709326 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..86aed946d7e30480109f0368b8f3db1266b71739 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"40-60%: The camera completes its circle around the locomotive, returning to the front view. It then begins to move towards the rear, capturing the back of the locomotive and the coupling mechanism. The orange traffic cones near the tracks are visible, and the camera continues to tilt and pan, ensuring all angles of the locomotive are shown. The background remains consistent with the hilly landscape and clear sky.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b32e843f3d9440cb7b990d34d8e74ee80620b590 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfafb417a1ee8dcbd3157207cd42d6148097093ce9a67d2d5c56ccf3d93f8044 +size 2431493 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..ca165b1ecb62faecae073c2485098da4530ee22b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"60-80%: The camera retraces its path along the side of the locomotive, moving back towards the front. It captures the same details as before, including the orange railings, steps, and the 'WESTERN PACIFIC' lettering. The camera then circles back to the front, providing a final comprehensive view of the locomotive from this angle. The background elements, such as the hills and blue sky, re-enter the frame consistently.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4ee2241654ba15798127cfa9e478d47bdf11cc5d --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8ada6cbe6a3c61698aa20cb8545b658b860c0f081cd83277495b532ab89f1f5 +size 1641853 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..6fed13ea8b3c985f01c2c7bd8f02d2bed7b968f2 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"80-100%: In the final segment, the camera continues to explore the locomotive from various angles, focusing on the front and side views. It captures the intricate details of the locomotive's structure, including the wheels, undercarriage, and safety features. The video concludes with a wide shot of the locomotive, emphasizing its position in the railway yard and the surrounding environment, maintaining spatial consistency and coherence throughout.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..80fbe2167b0a85f37b4a3f5eb9a400faca924e59 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4ccd4be7c0b4b0c346867148ee2d4aa4f0dcdd489f2c7c5615586267c2536db +size 2028382 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28578785cf70d499a71cd61a34e42027853dd694 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/console.log new file mode 100644 index 0000000000000000000000000000000000000000..7b851d46ddab404ef56f62d350f494c1fdf27bab --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/console.log @@ -0,0 +1,22 @@ +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:20:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:20:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:21:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/config.yaml +[02-19 19:21:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 19:21:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 19:21:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.json +[02-19 19:21:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:21:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 19:34:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 19:34:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 19:34:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 19:34:23|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6191ecf67637c5c48eec29549dbb6f409d818928 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/debug.log @@ -0,0 +1,353 @@ +[02-19 19:20:14|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 19:20:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:20:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:20:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 19:20:23|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 19:20:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 19:20:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 19:20:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 19:20:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 19:20:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 19:20:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 19:20:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 19:20:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 19:20:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 19:20:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 19:20:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 19:20:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 19:20:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 19:20:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 19:20:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 19:20:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 19:20:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 19:20:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 19:20:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 19:20:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 19:20:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 19:20:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 19:20:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 19:20:26|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 19:20:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 19:20:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 19:20:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 19:20:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 19:20:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 19:20:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 19:20:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:20:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:20:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 19:20:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:20:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:20:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:20:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 19:20:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:20:29|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 19:20:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:20:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:20:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 19:20:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.06 s +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:20:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:20:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:20:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:20:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 19:20:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 19:20:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.04 s +[02-19 19:20:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:21:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:21:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 19:21:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:21:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/config.yaml +[02-19 19:21:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 19:21:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 19:21:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="80-100%: The final segment shows the camera circling back to the side of the truck, mirroring the opening shot. The video concludes with a full view of the truck's side, emphasizing its vintage design and the surrounding urban environment, including the sidewalk, trees, and nearby buildings." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 19:21:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.json +[02-19 19:21:02|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:21:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 19:21:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4 +[02-19 19:21:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 19:21:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 19:21:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 19:21:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 19:34:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 19:34:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 19:34:21|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 19:34:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 19:34:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 19:34:23|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..99b88ffe0c2288fc33c7a7c5957a52ec4275ec87 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"0-20%: The video opens with a close-up of the truck's rear, showcasing its wooden bed and turquoise body. The camera begins a slow pan to the left, revealing the side profile of the truck and the 'San Pedro Square Market' logo. Background elements like yellow umbrellas and a street sign for 'St John' are visible.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..dd86e95f789d704eb8ef2ee74f7e82688a7350dc --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00d2f31e4e1d9a64813d200ce71db3d4dd4f3446ae7ecd4ebfb72b012d2f9ed8 +size 1427534 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..44ecb33a9e8f63e3f763ef93c495cfdf713de89e --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"20-40%: The camera continues its movement, circling the truck toward the front. The front grille and headlights come into view, while the background shifts to show more of the outdoor seating area and trees. The camera then starts retracing its path back toward the rear of the truck.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..11f20e00636d2989245ec7bd2784d2ba36d1a101 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:feced08e5fa6c96ce7e651c76d881e41c3a5775cfdd585b3a76616be51760744 +size 2195153 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..c19e248f1d88b507264688167562ef0079e984b1 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"40-60%: As the camera orbits back around the rear of the truck, it captures the same details seen earlier, such as the wooden bed and weathered paint. The background elements, including the yellow umbrellas and street sign, re-enter the frame as the camera moves. The truck remains stationary throughout.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a4cdc7f90bf7f8a68e8f451736e829b5fb0839a6 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c07053d022957e70352bdb9c9fb40700282790b0c71bf967a3423e3ad523199 +size 1711842 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..653cf8fb58ec13bcdd0b1e528cf803a7ffcf78cf --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"60-80%: The camera completes its orbit and returns to the front of the truck, providing a clear view of the grille and headlights. The background now includes a mix of the outdoor seating area and the street beyond. The camera then begins to pan back toward the side of the truck.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..524bf78b6d0001fd0d6dc4f4366d3af67fe0bae8 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0feed81a6bed28da6248676bdb98df880970ff18695512c4e2852b1db743282b +size 1688032 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..4beafc31ab3c5538f1866c296ae120d4e1be6815 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"80-100%: The final segment shows the camera circling back to the side of the truck, mirroring the opening shot. The video concludes with a full view of the truck's side, emphasizing its vintage design and the surrounding urban environment, including the sidewalk, trees, and nearby buildings.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23c85d7a8668aa67bab38f391e4e2598c1a08375 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cd07b68c8a1d351bf6392d8ae19e11353fc53c738b8248828208c09d312f4f49 +size 2395231 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7fef2a4fd91d751c4f9aadf20048e3036454b84c --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/console.log new file mode 100644 index 0000000000000000000000000000000000000000..915299c502d38ebe99bfa07a008493325ac8cec2 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/console.log @@ -0,0 +1,22 @@ +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:19:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:19:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:19:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/config.yaml +[02-19 19:19:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 19:19:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 19:19:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_4.json +[02-19 19:19:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:19:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 19:32:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 19:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 19:32:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 19:33:00|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..fbf5939ab3223389f9da8eb28a751b83b72b32ad --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/debug.log @@ -0,0 +1,353 @@ +[02-19 19:18:54|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 19:19:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:19:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 19:19:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 19:19:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 19:19:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 19:19:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 19:19:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 19:19:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 19:19:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 19:19:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 19:19:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 19:19:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 19:19:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 19:19:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 19:19:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 19:19:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 19:19:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 19:19:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 19:19:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 19:19:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 19:19:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 19:19:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 19:19:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 19:19:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 19:19:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 19:19:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 19:19:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 19:19:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 19:19:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 19:19:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 19:19:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 19:19:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 19:19:07|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 19:19:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 19:19:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 19:19:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 19:19:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 19:19:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 19:19:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 19:19:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:19:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 19:19:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:19:09|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 19:19:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 19:19:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 19:19:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 19:19:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 19:19:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 19:19:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 19:19:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 19:19:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 19:19:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 19:19:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.11 s +[02-19 19:19:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 19:19:41|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 19:19:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 19:19:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 19:19:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/config.yaml +[02-19 19:19:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 19:19:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 19:19:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final segment, the camera completes its loop around the Palace of Fine Arts, returning to the initial viewpoint. The video captures a panoramic view of the entire structure, integrating all previously explored spatial elements. The pond reflects the building, and the surrounding greenery frames the scene beautifully. The video concludes with a wide shot that highlights the harmonious blend of architecture and nature, providing a comprehensive overview of the Palace of Fine Arts.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 19:19:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_4.json +[02-19 19:19:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 19:19:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 19:19:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_3.mp4 +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 19:19:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 19:32:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 19:32:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 19:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 19:32:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 19:32:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 19:33:00|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..17f3294c57b2ad2070a0ee6c2c69d69c4d0217d6 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a wide shot of the Palace of Fine Arts, showcasing its domed structure and the reflective pond in the foreground. A bird flies across the clear blue sky, adding a sense of tranquility. The camera begins a slow horizontal pan to the right, revealing more of the surrounding greenery and the colonnade extending along the pond. As the pan progresses, the focus remains on the overall composition, with the pond reflecting the building and the sky.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e8b4c6c63e2b2047e333fc910d3a64888cfb9fed --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04870533915229289e0d0f6eb6d6e6e43e731cf2367965c63a0bdd4fd4850685 +size 718535 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..92149e7092cb24a5f9935ace80fc2e03c71135b7 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues its pan, it moves further along the pond's edge, capturing the intricate details of the columns and arches. The perspective shifts slightly, offering a closer view of the sculptural reliefs and the texture of the stone. The camera then starts to circle back toward the left side of the structure, maintaining a consistent distance from the building. The background re-enters the frame, with trees and distant structures becoming visible again.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9164fee5beee65d09a5b5c51fbdbaaf537656a94 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c85d158550dd90e426a938460cb7849313e5f2550b4b878ca8b6b30dd7bc097 +size 783822 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..a5f6b5d304d0ed90520aafec740559b0c1ed7bb8 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera retraces its path along the pond, revisiting the left side of the Palace of Fine Arts. This segment features a mix of wide shots and close-ups, alternating between the grandeur of the entire structure and the fine details of the architecture. The pond's surface reflects the building, creating a mirror-like effect. Fountain jets appear in the water, adding a dynamic element to the scene. The camera then begins to tilt upward, looking up at the towering columns and the ornate ceiling patterns.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3644127fff9070112988f9c75349be4022458e47 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d784f3fb182f7ee015b97687ee7f352029631947ddd56eafe14adf537ef4ca7 +size 1734093 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..84201fe4e49c8be2240b28b22d1edbf14a8ab068 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The video transitions to a series of vertical perspective shifts, with the camera looking up at the towering columns and the intricate ceiling designs. Close-up shots highlight the detailed carvings and sculptures adorning the arches and columns. The camera then pans downward, capturing the base of the columns and the surrounding greenery. The background elements, such as trees and distant buildings, re-enter the frame, maintaining visual consistency. The camera continues to circle the structure, revisiting earlier viewpoints and emphasizing the symmetry and balance of the design.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9e32e81e924c9fdced63c96079737cd7762318f --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a71ab2ecb166195d089715a7267a95b2f95a81bcf27136054204073694ec4ef +size 2537202 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..ad8ae5bd15ad877bdd8909c2831473ee973d64b3 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the camera completes its loop around the Palace of Fine Arts, returning to the initial viewpoint. The video captures a panoramic view of the entire structure, integrating all previously explored spatial elements. The pond reflects the building, and the surrounding greenery frames the scene beautifully. The video concludes with a wide shot that highlights the harmonious blend of architecture and nature, providing a comprehensive overview of the Palace of Fine Arts.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2bb33bf7045a7fb16948243fb279c639b6302e15 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84ab665e559900e10000273f2898a86ea73347a903aa411089d7f738fc5c9ea7 +size 1156867 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/config.yaml b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b6f4eb8ae6b378acd915f3a1dc92b530e2c0fc3c --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/console.log b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/console.log new file mode 100644 index 0000000000000000000000000000000000000000..cc83c79c7b43367c877e834048bf1ddce1a56363 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/console.log @@ -0,0 +1,22 @@ +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:23:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:23:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/config.yaml +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_4.json +[02-19 18:23:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:37:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:37:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:37:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:37:07|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/debug.log b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8b2fa2fda13b272ee715cc33cf27db6ca12c4df2 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/debug.log @@ -0,0 +1,353 @@ +[02-19 18:22:52|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 18:22:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:22:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 18:23:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 18:23:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 18:23:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 18:23:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 18:23:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 18:23:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 18:23:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 18:23:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 18:23:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 18:23:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 18:23:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 18:23:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 18:23:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 18:23:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 18:23:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 18:23:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 18:23:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 18:23:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 18:23:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 18:23:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 18:23:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 18:23:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 18:23:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 18:23:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 18:23:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 18:23:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 18:23:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 18:23:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 18:23:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 18:23:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 18:23:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 18:23:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 18:23:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 18:23:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 18:23:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 18:23:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 18:23:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 18:23:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 18:23:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:23:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 18:23:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:23:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 18:23:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 18:23:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 18:23:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 18:23:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.10 s +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 18:23:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.62 s +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 18:23:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 18:23:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 18:23:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 18:23:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 18:23:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.35 s +[02-19 18:23:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 18:23:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 18:23:40|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 18:23:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/config.yaml +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 18:23:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="The final segment revisits the rotunda's lower level, with a horizontal pan around the marble columns and mosaic floor, reinforcing the spatial coherence established earlier. The camera ascends once more to explore the stained-glass dome from multiple angles, capturing its vibrant colors and intricate designs. The video concludes with a smooth descent back to the ground level, retracing the path past the murals and railings, ensuring consistent alignment and orientation of these elements. The sequence loops back to the starting viewpoint, providing a comprehensive and cohesive exploration of the rotunda's interior architecture." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_4.json +[02-19 18:23:41|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_3.mp4 +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 18:23:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 18:23:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 18:23:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 18:37:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 18:37:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 18:37:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 18:37:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 18:37:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 18:37:07|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_4.mp4 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_0.json b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3f2d263f83167cec6f21aa2f7b1fe70a868e1ef8 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a horizontal pan around the rotunda's base, focusing on the marble columns and mosaic floor patterns. The camera begins its ascent, capturing the initial views of the stained-glass dome from a low angle, emphasizing the geometric designs and vibrant colors. As the pan continues, the viewer is introduced to the upper levels, where the ornate balcony railings and wall sconces come into view, setting the stage for the detailed exploration of the dome's architecture.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_0.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9b81e67d619d29ea58ea5a737e483606964c59a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f0589469122db3d863c567cd3904d687d0859e7534e73c682297bf997f49f5a +size 1945271 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_1.json b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..20d6e8afc1b33df41819c109cdb3a993c8fd9b57 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"In the second segment, the camera continues its upward movement, offering closer views of the stained-glass dome from various vertical perspectives. The intricate patterns and interplay of light become more pronounced as the camera moves higher. The scene transitions back to the ground level, retracing the path past the historical murals and ornate balcony railings, ensuring consistent alignment and orientation of these elements. The segment concludes with a smooth descent, preparing the viewer for the next phase of the exploration.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_1.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..10a0814689a7c614c22f45eebde823fe7e155aec --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8d2096666e1de8c138f03a06539e326eab6da3ad7c05c24d978c2bbc95b752d +size 2311580 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_2.json b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..d50d130caeac9596465963b555057077425623cf --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The third segment focuses on the detailed examination of the stained-glass dome from multiple angles, highlighting its elaborate designs and vibrant colors. The camera then descends back to the ground level, retracing the path past the murals and railings, ensuring consistent spatial relationships and lighting effects. The segment ends with a loop back to earlier viewpoints, particularly focusing on the interplay between the dome's patterns and the floor's geometry, while smoothly transitioning between different elevations.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_2.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2da979cd6924d93d3867560dc65bea91cd806e4a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0297ba264f70652887c285e0420c7051778fbd6175649bc8d24cb1ba6c695414 +size 1858064 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_3.json b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..1320d083e2699fab695d93a11ffe7e1fb4aaecd0 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"As the video progresses, the camera revisits earlier angles, particularly focusing on the interplay between the dome's patterns and the floor's geometry. The ascent to the dome's upper levels offers new perspectives, capturing the intricate designs and vibrant colors from various vertical angles. The descent back to the ground level retraces the path past the murals and railings, ensuring consistent alignment and orientation of these elements. The segment concludes with a smooth transition back to the initial viewpoints, reinforcing the cohesive exploration of the rotunda's interior.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_3.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bbef2ecd09f498b858c1317485804c3f49cb4d28 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63aa86b3de7486fa1d9a5127786cb8378d0925041741bd3c6f381e2357ab76fc +size 2336023 diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_4.json b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..bf5d183882059e71cf3157106c95734ad9a046e1 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The final segment revisits the rotunda's lower level, with a horizontal pan around the marble columns and mosaic floor, reinforcing the spatial coherence established earlier. The camera ascends once more to explore the stained-glass dome from multiple angles, capturing its vibrant colors and intricate designs. The video concludes with a smooth descent back to the ground level, retracing the path past the murals and railings, ensuring consistent alignment and orientation of these elements. The sequence loops back to the starting viewpoint, providing a comprehensive and cohesive exploration of the rotunda's interior architecture.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_4.mp4 b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b89605815085d70949d9aa8d34bf636644025b7a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:026ca161b1cd4723c62f0f0e17289cb277058037db22ae41ac452bc3766f33c1 +size 2674021 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..393e720f9f5ac9d357c1ca7644434f6faf67234a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bbc8318b3fb8305870062064649971854b3ac3fd5f5a1c8d60c05e43fef7cc3a +size 6891269 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1e6d03b404fc1eeafb9e088ffac1a75fd458186a --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d622959bc6a1dd245f4bb6a5c5e991331f7ea05b928be983424532a8e6e19e15 +size 7370354 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3802976f91f1c5ab2ec6eeec927570a8881f5255 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9f60be77507c4e54dfa1eacc4a6b4bb5c944559541e6e83caeec7dc4475ca46b +size 6374473 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cb1db69caf208759dbd26d7135247638247f418b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_2114a5ba3a1cca5645c63f9019c21558f5c4f2573109913bf20b6c7ea56b1d3d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a622f54c53716527c847e5cecfa5298d264adb70b9a75fa7f0ea6f587386297c +size 7313588 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8ca506e785c34250ff2aea2ecc6a0464b6bd0966 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4d64ae1700a7d97b5f46e0ea623778bdbbf725eac172db0a5ea884e9f0c1a81 +size 7049198 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b8506fb198ef8375780aa6da00a0600cca5b2652 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e16bd340b4476abfc9b09dc16fe7b88c0da484ce8be7db9ea3aedbb050571a8 +size 7382049 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..17a3b00baaf68c0f02e6a1c484248cd54591423d --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:444f9f6858e4c472a474e4fd515a2dd20e50973a1477dfc7aab2b4128b2a123b +size 6896864 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..342877a86faf820137b1c3e36bf43982f54f4507 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_5817f81eddb995fc6004532115c631a2eba651d7af9e085afc3210a19e0e1904.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ad2c66a1342de9987c6b5cdd79653742416705e13744fda1b8d07a022f5f386 +size 9463451 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa82be354c28531fff1251c4b6eabb229a4a11c9 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c25c5029b1c8d2390ce0a3d86f53c9bf6e56bcdc9298016ab463c9b6e2c0fac +size 8609794 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5268a2793720a75aab95bca32828422d8d9e6f24 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c7b69074341853fd071aa56fe06851887156cde27a41f3925300cf83c91e551 +size 6071173 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bd2d295378a6a4a82e15054aa5d5fdd4c3ce39bb --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afdd412bd484a56c50289e306e287d397974eb77ac2aa114e50333fe33d3b6d0 +size 8155050 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b5ec43e270c88f5ff1b8136efd5047e82094960 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_8565d4650361a97103a6970031ff0e8a23c498d1dde1b985a664d1151c99aed0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2f0eac949d797faf19e7df7c29a61487c62f15a4070507e1cb8b0d537f7e4445 +size 9044455 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77b80e78a95ceb9418b7a0feec09b73a13360688 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6c7add6f56145e09c92e4fc20b349b1a1b85c00982eb28b2d11cf102168483e +size 9321783 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f6c8cf1a062bde18beda79e6b0fe573334421cea --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:75838243390ce510760ce82d6dde9d7e9424798a21e3056592b7762502a6ca02 +size 8843213 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3e6618e5622cd94c57f05e679b995083b801667b --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5528c2d7fa4d4a30e4f0699439ae49499a8862155c48937d6cc9e5229b679da +size 7323937 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6ad9a34139cf957e69ac0cd1163f8dbe32a56327 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7abae1ff350f0d7b9d4c5c05bf54262d5655515883a0b3c44da179f48d22940c +size 1058407 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..605f2b92401087e13340495925b21757d4c605a0 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_bb7e1def1dc7dfb3ce3a3d0aae0af412104bb820faaf0a5617451bd6d147bf1a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c798c2947ac43ccbb8b77a590a7e4432a952105905b3c087ef4fa203dcbe2422 +size 9159404 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..07e94717a1d59dd7de6601b0f5aac15ae2b2d0a8 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a460f76d102a0c6467be6021a15752349739ddf98fe2511aec8dff0c70327a88 +size 9414647 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6d3287db4b31bb001892cfb0d2765984d89f3e42 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_d44bf0fee21c2dfd0bb99b5c4e938ca645a4153733bb0df1169fa482f41ba0ef.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f65ab7e8d476027692936dcb9cfde819090c1e382028d5304721d3bb38d6e454 +size 6927424 diff --git a/0219155836_cosmos_tnt_spatial/videos/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2.mp4 b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9f7b8059cba0c90b2fbcb95b31d372c60ee3e032 --- /dev/null +++ b/0219155836_cosmos_tnt_spatial/videos/mem_tnt_f407eca1d989587c5c350fce42156c235b6e17dedc08c1013fa739d1088d83f2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:46e51b893a5d411dc0354b154d5cb098e171183f967e0b91799e1d0d1d17b061 +size 11121844