diff --git a/.gitattributes b/.gitattributes index 183a89f936ae14fb182851ee4fbd2a099ea10e02..14979ec3313f7f843d5ac3f736992f65b12df522 100644 --- a/.gitattributes +++ b/.gitattributes @@ -61,3 +61,71 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text 0219122156_cosmos_mip_spatial/videos/mem_mipnerf360_575f06997f088dc5824bec5a960459303edd0154e65b3594e29810380ac7ea50.mp4 filter=lfs diff=lfs merge=lfs -text 0219122156_cosmos_mip_spatial/videos/mem_mipnerf360_5f84c46d31489bd1a6b57b7c5c8cbc15280e279f8eb03122abd9362ac7333d13.mp4 filter=lfs diff=lfs merge=lfs -text 0219122156_cosmos_mip_spatial/videos/mem_mipnerf360_a8f137cd302ac277409b1dcd26e4c8a03d8600bc4f6a80af40c71a5a980a98c5.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.mp4 filter=lfs diff=lfs merge=lfs -text +0219133414_cosmos_tnt_object/videos/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.lock new file mode 100644 index 0000000000000000000000000000000000000000..eaa56ebb21a169118ae4c884fff03e184e1a2103 --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470848 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.lock new file mode 100644 index 0000000000000000000000000000000000000000..b9a29d8802e28ba6ed0ac1358a5c303acc2e685a --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470853 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.lock new file mode 100644 index 0000000000000000000000000000000000000000..6aa7e13062aa9060d251fb3ddbb296fa68d1c8a6 --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470852 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.lock new file mode 100644 index 0000000000000000000000000000000000000000..c3999a9ae325de260b3ba935c7b199e1f664953f --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470849 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.lock new file mode 100644 index 0000000000000000000000000000000000000000..6c6e7c9a5fee16b4b627aa6ac4d1e230ac6849ea --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470851 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.lock new file mode 100644 index 0000000000000000000000000000000000000000..96a1a8ce8169a843313ae073afb706b1ab312675 --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470850 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.lock new file mode 100644 index 0000000000000000000000000000000000000000..b9a29d8802e28ba6ed0ac1358a5c303acc2e685a --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470853 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.lock new file mode 100644 index 0000000000000000000000000000000000000000..eaa56ebb21a169118ae4c884fff03e184e1a2103 --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470848 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.lock new file mode 100644 index 0000000000000000000000000000000000000000..c3999a9ae325de260b3ba935c7b199e1f664953f --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470849 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.lock new file mode 100644 index 0000000000000000000000000000000000000000..9b2008e3225a80c5a10bd2e8d6a1af4260ceb024 --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470847 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.lock new file mode 100644 index 0000000000000000000000000000000000000000..6c6e7c9a5fee16b4b627aa6ac4d1e230ac6849ea --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470851 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.lock new file mode 100644 index 0000000000000000000000000000000000000000..96a1a8ce8169a843313ae073afb706b1ab312675 --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470850 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.lock new file mode 100644 index 0000000000000000000000000000000000000000..6aa7e13062aa9060d251fb3ddbb296fa68d1c8a6 --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470852 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/.locks/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.lock b/0219133414_cosmos_tnt_object/.locks/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.lock new file mode 100644 index 0000000000000000000000000000000000000000..9b2008e3225a80c5a10bd2e8d6a1af4260ceb024 --- /dev/null +++ b/0219133414_cosmos_tnt_object/.locks/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.lock @@ -0,0 +1 @@ +done:bravo-tongclass-a800:2470847 \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/gpu_0.log b/0219133414_cosmos_tnt_object/gpu_0.log new file mode 100644 index 0000000000000000000000000000000000000000..2f088d057d43ba59d2e34b09522ab1b5195c36b9 --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_0.log @@ -0,0 +1,4 @@ +[lock] Claimed: mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f (worker=bravo-tongclass-a800:2470853) +[lock] Done: mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f +[lock] Claimed: mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0 (worker=bravo-tongclass-a800:2470853) +[lock] Done: mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0 diff --git a/0219133414_cosmos_tnt_object/gpu_0_instances.txt b/0219133414_cosmos_tnt_object/gpu_0_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..41aced131d74520bf6cec11d0c0e4f5495cb1ae0 --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_0_instances.txt @@ -0,0 +1,2 @@ +mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f +mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0 diff --git a/0219133414_cosmos_tnt_object/gpu_1.log b/0219133414_cosmos_tnt_object/gpu_1.log new file mode 100644 index 0000000000000000000000000000000000000000..c3f5596303cb1c0543103ffa3a7627cd948b43c9 --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_1.log @@ -0,0 +1,4 @@ +[lock] Claimed: mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e (worker=bravo-tongclass-a800:2470851) +[lock] Done: mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e +[lock] Claimed: mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b (worker=bravo-tongclass-a800:2470851) +[lock] Done: mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b diff --git a/0219133414_cosmos_tnt_object/gpu_1_instances.txt b/0219133414_cosmos_tnt_object/gpu_1_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..1aee12c864903baaf4a223928ed636a9d944c96b --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_1_instances.txt @@ -0,0 +1,2 @@ +mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e +mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b diff --git a/0219133414_cosmos_tnt_object/gpu_2.log b/0219133414_cosmos_tnt_object/gpu_2.log new file mode 100644 index 0000000000000000000000000000000000000000..66526d14d149abb62fcd71d717243d41e6c9772c --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_2.log @@ -0,0 +1,4 @@ +[lock] Claimed: mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f (worker=bravo-tongclass-a800:2470850) +[lock] Done: mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f +[lock] Claimed: mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857 (worker=bravo-tongclass-a800:2470850) +[lock] Done: mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857 diff --git a/0219133414_cosmos_tnt_object/gpu_2_instances.txt b/0219133414_cosmos_tnt_object/gpu_2_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..9d3715a0f838ce68eb444574234f8a2df722b49a --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_2_instances.txt @@ -0,0 +1,2 @@ +mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f +mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857 diff --git a/0219133414_cosmos_tnt_object/gpu_3.log b/0219133414_cosmos_tnt_object/gpu_3.log new file mode 100644 index 0000000000000000000000000000000000000000..73989c7384abac306fcc7754aa13efdddadda2b1 --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_3.log @@ -0,0 +1,4 @@ +[lock] Claimed: mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017 (worker=bravo-tongclass-a800:2470849) +[lock] Done: mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017 +[lock] Claimed: mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de (worker=bravo-tongclass-a800:2470849) +[lock] Done: mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de diff --git a/0219133414_cosmos_tnt_object/gpu_3_instances.txt b/0219133414_cosmos_tnt_object/gpu_3_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..ce715ee46e2a94d475cf859a5acf0ec768f3490e --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_3_instances.txt @@ -0,0 +1,2 @@ +mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017 +mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de diff --git a/0219133414_cosmos_tnt_object/gpu_4.log b/0219133414_cosmos_tnt_object/gpu_4.log new file mode 100644 index 0000000000000000000000000000000000000000..25df3164fcd4c29c2eff47f6b136e536641ae284 --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_4.log @@ -0,0 +1,4 @@ +[lock] Claimed: mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144 (worker=bravo-tongclass-a800:2470852) +[lock] Done: mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144 +[lock] Claimed: mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1 (worker=bravo-tongclass-a800:2470852) +[lock] Done: mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1 diff --git a/0219133414_cosmos_tnt_object/gpu_4_instances.txt b/0219133414_cosmos_tnt_object/gpu_4_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..db5e33ec892f4f499bf7781a3744f9a21050563a --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_4_instances.txt @@ -0,0 +1,2 @@ +mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144 +mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1 diff --git a/0219133414_cosmos_tnt_object/gpu_5.log b/0219133414_cosmos_tnt_object/gpu_5.log new file mode 100644 index 0000000000000000000000000000000000000000..9495c3ce1f11a0b4f57318108e51ff820810e595 --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_5.log @@ -0,0 +1,4 @@ +[lock] Claimed: mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc (worker=bravo-tongclass-a800:2470848) +[lock] Done: mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc +[lock] Claimed: mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5 (worker=bravo-tongclass-a800:2470848) +[lock] Done: mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5 diff --git a/0219133414_cosmos_tnt_object/gpu_5_instances.txt b/0219133414_cosmos_tnt_object/gpu_5_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..507a84e942dadb381dd102e48147e51860f0a186 --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_5_instances.txt @@ -0,0 +1,2 @@ +mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc +mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5 diff --git a/0219133414_cosmos_tnt_object/gpu_6.log b/0219133414_cosmos_tnt_object/gpu_6.log new file mode 100644 index 0000000000000000000000000000000000000000..509d41f587b6952d24c6fd3315b74b8f1dacb334 --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_6.log @@ -0,0 +1,4 @@ +[lock] Claimed: mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21 (worker=bravo-tongclass-a800:2470847) +[lock] Done: mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21 +[lock] Claimed: mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1 (worker=bravo-tongclass-a800:2470847) +[lock] Done: mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1 diff --git a/0219133414_cosmos_tnt_object/gpu_6_instances.txt b/0219133414_cosmos_tnt_object/gpu_6_instances.txt new file mode 100644 index 0000000000000000000000000000000000000000..eab2394fe1a2d8a0bfc7827c2157eecae2c4a403 --- /dev/null +++ b/0219133414_cosmos_tnt_object/gpu_6_instances.txt @@ -0,0 +1,2 @@ +mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21 +mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1 diff --git a/0219133414_cosmos_tnt_object/mapping.json b/0219133414_cosmos_tnt_object/mapping.json new file mode 100644 index 0000000000000000000000000000000000000000..20fcd4f8c569388f8aabc75589a573af1655a504 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mapping.json @@ -0,0 +1,358 @@ +[ + { + "instance_id": "mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ] + } + }, + { + "instance_id": "mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + }, + { + "instance_id": "mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0", + "fps": 16.0, + "height": 704, + "width": 1280, + "model_name": "cosmos", + "level": "level_3", + "prompts_mapping": { + "prompt_1": [ + 0, + 76 + ], + "prompt_2": [ + 77, + 153 + ], + "prompt_3": [ + 154, + 230 + ], + "prompt_4": [ + 231, + 307 + ], + "prompt_5": [ + 308, + 384 + ] + } + } +] \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01acc0c25805610b1727588c2657a6cbbdb26f93 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/console.log b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/console.log new file mode 100644 index 0000000000000000000000000000000000000000..1035ba85bdd79fb5bfb49b27279f81db612ab8ab --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/console.log @@ -0,0 +1,22 @@ +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:46:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:46:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/config.yaml +[02-19 14:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.json +[02-19 14:46:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:59:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:59:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:59:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:00:01|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e0ab4198d0bdc245cdcc52fee36dcc009a35f920 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/debug.log @@ -0,0 +1,353 @@ +[02-19 14:45:55|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 14:46:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:46:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:46:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 14:46:04|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 14:46:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 14:46:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 14:46:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 14:46:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 14:46:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 14:46:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 14:46:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 14:46:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 14:46:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 14:46:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 14:46:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 14:46:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 14:46:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 14:46:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 14:46:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 14:46:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 14:46:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 14:46:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 14:46:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 14:46:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 14:46:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 14:46:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 14:46:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 14:46:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 14:46:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 14:46:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 14:46:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 14:46:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 14:46:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 14:46:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 14:46:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:46:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:46:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 14:46:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:46:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:46:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 14:46:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:46:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 14:46:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:46:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:46:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.12 s +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:46:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:46:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 14:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 14:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.53 s +[02-19 14:46:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:46:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 14:46:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/config.yaml +[02-19 14:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:46:43|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='The video concludes with a final side view of the tank, reinforcing its identity and structural consistency within the hangar setting. The camera captures the intricate details of the track system and road wheels, ensuring identity persistence and texture fidelity. The spacious hangar environment provides a fitting backdrop for the detailed exploration of this military tank.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 14:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.json +[02-19 14:46:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:46:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4 +[02-19 14:46:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 14:46:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 14:46:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 14:46:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 14:59:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 14:59:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:59:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:59:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:00:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 15:00:01|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..af6262a3d26f0196545cd3678561a880ffca6e87 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a side view of the camouflaged Pz IV Ausf. H tank, focusing on its six large road wheels and track system. The red number '501' is clearly visible on the turret, and the black cross insignia can be seen on the hull. The camera begins to pan around the tank, gradually revealing more of its structure and details.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..136b006e04643fad815ba1245d1977ab4c33790e --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e6e2d118c15f8e81b346349f823d64797688ab036e66748d6e24b7418edb9a1 +size 991504 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.json b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..62b59f59fdc225a17f60bf17ee6199f121fcef40 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues to move, it transitions to the front view of the tank, highlighting the hull's design, the main gun barrel, and the black cross insignia. The intricate details of the mounted equipment and the camouflage pattern are captured in detail. The camera then starts to move toward the rear of the tank, showing the engine deck and exhaust systems.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..86784b48c724bbe5db86e196b632aafae4fb11ad --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36fbe808efd56cdfb5fbc212d8632c3865b60751750efde95e0eca549b990cdd +size 1533373 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.json b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..affa5a522c3c16a0abc698dee88a7c5c17861d5e --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The middle section of the video focuses on the rear view of the tank, providing a clear look at the engine deck, exhaust pipes, and additional mounted equipment. The consistent camouflage pattern and fine details such as rivets and hatches are maintained, ensuring geometric integrity and texture fidelity. The spacious hangar environment allows for clear visibility of these features.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..077aa6b3db2d88d0a8ccfd0e76ecb1198918b34a --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:799422ae8b3bdc7f8f7de82324c6d105cd3a654acb648d494cf7d84054fb506c +size 1099837 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.json b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..355089d8c3468d5d5b1a64c1bf78c594339ec095 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the latter part of the video, the camera captures close-up views of the tank from the front and rear, emphasizing the hull's design, the main gun barrel, and the engine deck. The red number '501' and black cross insignia remain prominent, and the camera highlights the tank's complex surface textures and mounting details. The video ensures semantic correctness by presenting the tank from all necessary angles.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9d75de7026c52012850f34fd692ef5c6ca940394 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ad076f620421806592d7a60e49bd8826d512db7c8aea8613f28c6d678cebd13 +size 2032215 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.json b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..e9611b7f874b55a20ab11a063efaf42a845f359e --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with a final side view of the tank, reinforcing its identity and structural consistency within the hangar setting. The camera captures the intricate details of the track system and road wheels, ensuring identity persistence and texture fidelity. The spacious hangar environment provides a fitting backdrop for the detailed exploration of this military tank.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b23546e5423afe7a3ec4945ae2e4f2ac2821625c --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77b92dacad877d62e15cdb60c3743506d271da1ea1a5bc418a55d87c7c828099 +size 1532698 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f6535356fc7cddf83550ad9633c6ab9bda00d50 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/console.log b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/console.log new file mode 100644 index 0000000000000000000000000000000000000000..8f395a369ddb64660234e19e00f232893ee54936 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/console.log @@ -0,0 +1,22 @@ +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 15:44:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 15:44:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 15:45:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/config.yaml +[02-19 15:45:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 15:45:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 15:45:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.json +[02-19 15:45:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 15:45:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 15:58:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 15:58:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 15:58:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:58:26|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3ef1a0b944273c1af2e2de0d5859440b94d98e30 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/debug.log @@ -0,0 +1,353 @@ +[02-19 15:44:11|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 15:44:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 15:44:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 15:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 15:44:21|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 15:44:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 15:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 15:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 15:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 15:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 15:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 15:44:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 15:44:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 15:44:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 15:44:21|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 15:44:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 15:44:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 15:44:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 15:44:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 15:44:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 15:44:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 15:44:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 15:44:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 15:44:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 15:44:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 15:44:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 15:44:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 15:44:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 15:44:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 15:44:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 15:44:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 15:44:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 15:44:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 15:44:24|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 15:44:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 15:44:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 15:44:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 15:44:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 15:44:24|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 15:44:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 15:44:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 15:44:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 15:44:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 15:44:26|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 15:44:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 15:44:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 15:44:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 15:44:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.63 s +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:44:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 15:44:28|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.68 s +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 1.17 s +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 1.18 s +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 1.18 s +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 15:44:29|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 15:44:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 15:44:55|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 15:44:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.59 s +[02-19 15:44:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 15:45:00|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:00|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:45:01|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 15:45:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 15:45:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/config.yaml +[02-19 15:45:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 15:45:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 15:45:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="In the final segment, the camera provides a sweeping view of the courtroom from different angles, capturing the rows of dark wooden benches with green upholstered chairs arranged neatly on the red patterned carpet. The camera pans across the room, revealing the ornate ceiling with elaborate chandeliers and a large, circular stained-glass dome that serves as the centerpiece. The video concludes with a focus on the judge's bench, the American and state flags, and the classical portraits on the walls, emphasizing the consistent theme of the courtroom's grandeur and historical significance." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 15:45:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.json +[02-19 15:45:01|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 15:45:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 15:45:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4 +[02-19 15:45:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 15:45:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 15:45:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 15:45:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 15:58:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 15:58:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 15:58:23|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 15:58:23|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:58:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 15:58:26|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..6bd36c6d92e305f0bfa41140b50fc86ca1480812 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a wide shot of the courtroom, showcasing the rows of dark wooden benches with green upholstered chairs set against the backdrop of a red patterned carpet. The camera pans to the right, revealing the ornate ceiling with elaborate chandeliers and a large, circular stained-glass dome that dominates the space. The initial frames also capture the judge's bench, the American and state flags, and the classical portraits on the walls, setting the stage for the detailed exploration of the courtroom's architecture and furnishings.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b5a0de4b5975321c2d75eeaddb92c2ff87fbd2af --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ba509c3824b0d7adb556a3174929e78ece02f85d4b1408053ce9baa94198608 +size 1601385 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.json b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..753f24a5f2aa58cd7c7c1364dd16435f2e2b1afe --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the camera moves to the left side of the courtroom, focusing on a wooden display cabinet and a series of windows with white blinds. The movement continues upward, capturing the intricate patterns of the stained-glass dome ceiling, emphasizing the vibrant colors and floral motifs. The camera then shifts to the front of the courtroom, providing a closer view of the judge's bench and the two flags, maintaining the consistent theme of the courtroom's grandeur and historical significance.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..69651404c401f163477c9c0a20611105f57ca1a1 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ab090475ffc4a0c41895a08b9f9f493f539b96312fe2794885a410c4ca67af8 +size 1848465 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.json b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..0214c205bef009ee29e5c19a857d622ff3677df8 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"In the third segment, the camera explores the back of the courtroom, highlighting the rows of empty wooden benches and the red carpet. The perspective shifts to the left side of the room, showcasing a wooden display cabinet and a series of windows with white blinds. The camera then ascends to capture the detailed patterns of the stained-glass dome ceiling, emphasizing the intricate details and the play of light through the glass. The focus returns to the front of the courtroom, where the judge's bench and the two flags are prominently displayed, reinforcing the consistent theme of the courtroom's design and features.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bade6ea70a8212ec1687b14c6e42773c6456370b --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2dc7acd5f221f67f652a381c5d88292129b7a8c6f10f40583b176189ed460b26 +size 1571209 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.json b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..8ad614df2975cafc219c3caec97edb94f13cee39 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The fourth segment continues with a closer look at the judge's bench and the surrounding area, including the green upholstered chair and the American flag. The perspective shifts to the right side of the courtroom, revealing more of the wooden benches and the red carpet. The camera then moves upward again to capture the stained-glass dome ceiling, highlighting the intricate details and the play of light through the glass. The final scenes return to the front of the courtroom, offering a comprehensive view of the entire space, reinforcing the consistency and coherence of the courtroom's design and features.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2af4c99aca43174279e7aff1e65b42d0e08e60ae --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:132014a2f0beb6ff1101e5b8c34449855ba61ff821e59cce951d24082ce6ce38 +size 1275634 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.json b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..8d3e558226ca972c89682c4f9d9921eb4de8755e --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the camera provides a sweeping view of the courtroom from different angles, capturing the rows of dark wooden benches with green upholstered chairs arranged neatly on the red patterned carpet. The camera pans across the room, revealing the ornate ceiling with elaborate chandeliers and a large, circular stained-glass dome that serves as the centerpiece. The video concludes with a focus on the judge's bench, the American and state flags, and the classical portraits on the walls, emphasizing the consistent theme of the courtroom's grandeur and historical significance.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3942347d31998e7b3da5d765e3afa2ff72f308d9 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b45573d584bacfde8e4807ed49e585695064f4db849f512641c44ca42d153951 +size 1566241 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2da7d370f2af3a2f62d72d6a128fdec656b35a25 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/console.log b/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/console.log new file mode 100644 index 0000000000000000000000000000000000000000..5f4a5709e0025e4b17b0d255a48897ffc01d9004 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/console.log @@ -0,0 +1,22 @@ +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:51:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:51:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:51:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:51:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/config.yaml +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.json +[02-19 14:52:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 15:06:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 15:06:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 15:06:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:06:28|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0b243e4f397840180a341dc6a2b3f510ce8137e7 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/debug.log @@ -0,0 +1,352 @@ +[02-19 14:51:18|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 14:51:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:51:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:51:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 14:51:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 14:51:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 14:51:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 14:51:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 14:51:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 14:51:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 14:51:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 14:51:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 14:51:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 14:51:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 14:51:27|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 14:51:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 14:51:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 14:51:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 14:51:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 14:51:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 14:51:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 14:51:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 14:51:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 14:51:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 14:51:28|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 14:51:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 14:51:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 14:51:29|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 14:51:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 14:51:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 14:51:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 14:51:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 14:51:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 14:51:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 14:51:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 14:51:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:51:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:51:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 14:51:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:51:30|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:51:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:51:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 14:51:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:51:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 14:51:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:51:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:51:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 14:51:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:51:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.15 s +[02-19 14:51:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 14:51:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 14:51:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 14:51:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:51:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 14:51:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 14:51:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 14:51:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 14:51:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 14:51:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 14:51:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 14:51:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 14:51:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 14:51:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:51:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:52:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 14:52:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 14:52:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.61 s +[02-19 14:52:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:52:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:52:06|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/config.yaml +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 14:52:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with a close-up frontal shot of the military tank, highlighting its olive green paint, signs of rust, and the identification numbers '1 ATB 4-13A' and '4-404' on the front. The camera then begins to move, panning to the right to reveal the tank's side profile, focusing on the long cannon barrel, the intricate track system with numerous road wheels, and the turret equipped with ropes and mesh baskets. As the camera continues its movement, it captures the rear and top views of the tank, showing the side skirts, rear hull details, and additional mounted equipment. The perspective then shifts back to the front, providing a full frontal view that encompasses the tank's entire structure and condition, while also showing the interior of the hangar with other tanks and military vehicles in the background. Finally, the camera moves around the tank once more, offering a detailed look at its various components and features from different angles." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.json +[02-19 14:52:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 14:52:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 14:52:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 15:06:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 15:06:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 15:06:25|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 15:06:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:06:27|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 15:06:28|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..4beb4c24d2438011e99d7b4e37d04930edb80b32 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up frontal shot of the military tank, highlighting its olive green paint, signs of rust, and the identification numbers '1 ATB 4-13A' and '4-404' on the front. The camera then begins to move, panning to the right to reveal the tank's side profile, focusing on the long cannon barrel, the intricate track system with numerous road wheels, and the turret equipped with ropes and mesh baskets. As the camera continues its movement, it captures the rear and top views of the tank, showing the side skirts, rear hull details, and additional mounted equipment. The perspective then shifts back to the front, providing a full frontal view that encompasses the tank's entire structure and condition, while also showing the interior of the hangar with other tanks and military vehicles in the background. Finally, the camera moves around the tank once more, offering a detailed look at its various components and features from different angles.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c1cd7cd01343d239887d7ce4b7dba4fba2d0a220 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ea923628fc01aca82cfe84d99d79543ae022e7eb213490dd4fa31fa189424ad +size 1872960 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c19b139aad3e91b55f933a8fb5b863642494ef9 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/console.log b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/console.log new file mode 100644 index 0000000000000000000000000000000000000000..301ecc6b451bcf1ef9a862aa6063e18080f08b5f --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/console.log @@ -0,0 +1,22 @@ +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:32:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:32:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/config.yaml +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.json +[02-19 14:32:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:45:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:45:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:45:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 14:45:59|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f537480d8a38fab646e6f5e45301fd9eaa29759a --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/debug.log @@ -0,0 +1,353 @@ +[02-19 14:31:50|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 14:31:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:31:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 14:31:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 14:31:59|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 14:32:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 14:32:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 14:32:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 14:32:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 14:32:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 14:32:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 14:32:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 14:32:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 14:32:01|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 14:32:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 14:32:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 14:32:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 14:32:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 14:32:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 14:32:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 14:32:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 14:32:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 14:32:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:32:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:32:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 14:32:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:32:03|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:32:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:32:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 14:32:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:32:04|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 14:32:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:32:04|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:32:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 14:32:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:32:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.02 s +[02-19 14:32:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 14:32:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 14:32:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 14:32:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:32:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:32:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 26.97 s +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:32:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:36|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:37|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/config.yaml +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:32:37|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="In the final 80-100% segment, the camera continues its slow orbit around the statue, now focusing on the base and the circular stone fountain plaza. The intricate details of the statue's base, including any inscriptions, are highlighted. The camera then moves slightly closer to the statue, providing a more detailed view of the bronze material and the greenish patina in the crevices. The video concludes with a comprehensive view of the statue, emphasizing its identity persistence, geometric integrity, and texture fidelity from all angles, while the serene garden setting remains a constant backdrop." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.json +[02-19 14:32:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4 +[02-19 14:32:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 14:32:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 14:32:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 14:32:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 14:45:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 14:45:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:45:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:45:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 14:45:58|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 14:45:59|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..6ccdcdcf8abe1483d1423d0fba0368f879c86605 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"At the beginning of the video, the camera presents a frontal view of the bronze statue of a seated man reading a book. The statue, with its greenish patina and classical robes, is situated in a circular stone fountain plaza. The surrounding area is lush with green grass, mature trees, and colorful flowering beds. The camera starts to move slowly to the right, beginning to reveal the side profile of the statue and the elegant building in the background, which features multiple windows and a tiled roof.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2e69b981cc31f4e1ddd08b72500cfef95c04efb8 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bf9ac94aed8eb591553783be0dbbd763fbc8a308559d1b853958c23bf71f4698 +size 1338824 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.json b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..2c807917457dc84bee9f81303877cfcb6d06402b --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video transitions into the 20-40% segment, the camera continues its smooth orbit around the statue, now showing it from a side angle. The intricate details of the statue's robes and the open book on its lap become more apparent. The background remains consistent, with the green grass, trees, and flowering beds providing a picturesque setting. The camera movement is steady, ensuring that the statue's geometric integrity is preserved, and the viewer can appreciate the fine textures of the bronze material.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7b6145e149d02b6365ea9a20f834dfb4875c07c1 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4801cd30b627bbc119c8091925666481c3c7cf00b4a14dd032b125a23fe1b5f4 +size 1120127 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.json b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..80ac6e791b45f7263ade77af77997db57c68e127 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"In the 40-60% segment, the camera moves further around the statue, transitioning from the side to the rear view. The statue's back is now visible, showcasing the detailed sculpting of the robes and the overall form of the seated figure. The circular stone fountain plaza is still prominent, and the surrounding greenery continues to frame the scene beautifully. The large building in the background remains partially obscured by the trees, adding depth to the composition. The camera's movement is fluid, maintaining a clear and consistent focus on the statue.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..405da6258a3a5b1b21e4215e1b1e681cc5d36a3f --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a10db124bae234609665542abc9443c76156bb585238278a7cc15cfafd67bb5 +size 1001400 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.json b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..55a3384a71c355c494cd3f2b90737eca694a36cb --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"As the video progresses into the 60-80% segment, the camera begins to move back towards the front of the statue, completing its orbit. The viewer is once again presented with a frontal view of the statue, similar to the opening shot. The intricate details of the statue's face, the book in its hands, and the rock-like base are clearly visible. The surrounding environment, including the circular stone plaza, green grass, and flowering beds, remains unchanged, providing a cohesive and serene setting.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8944bd60f4273c2193955f786b03e06ddac752f0 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:271e7248a31df814ac208358168dc065ee44f329bf6691f43421c105dd43ba71 +size 1008009 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.json b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..7a9774637aee7363bd28855c764e917393f587c6 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final 80-100% segment, the camera continues its slow orbit around the statue, now focusing on the base and the circular stone fountain plaza. The intricate details of the statue's base, including any inscriptions, are highlighted. The camera then moves slightly closer to the statue, providing a more detailed view of the bronze material and the greenish patina in the crevices. The video concludes with a comprehensive view of the statue, emphasizing its identity persistence, geometric integrity, and texture fidelity from all angles, while the serene garden setting remains a constant backdrop.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a3b1669d333ffbed681e4eb2023e386e851f441c --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d00726b6481287ea7673116c366ccc1e7c8c2f6a1ee76fb2cda7a38d01f4471 +size 1375193 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb93f8f5c8c1c56a4563b8a9a1b2aedace7999f --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/console.log b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/console.log new file mode 100644 index 0000000000000000000000000000000000000000..3247e9a5d781cf718c5dedb42b10cba4fc09f22b --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/console.log @@ -0,0 +1,22 @@ +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:32:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/config.yaml +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.json +[02-19 14:33:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:46:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:46:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:46:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 14:46:43|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b1baa228c886f5c408ff2233dff8f5366aee5788 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/debug.log @@ -0,0 +1,353 @@ +[02-19 14:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 14:32:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:32:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 14:32:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 14:32:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 14:32:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 14:32:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 14:32:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 14:32:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 14:32:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 14:32:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 14:32:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 14:32:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 14:32:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 14:32:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 14:32:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 14:32:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 14:32:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 14:32:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 14:32:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 14:32:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 14:32:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 14:32:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 14:32:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 14:32:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 14:32:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 14:32:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 14:32:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 14:32:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 14:32:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 14:32:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 14:32:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 14:32:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 14:32:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 14:32:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 14:32:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:32:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:32:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 14:32:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:32:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:32:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:32:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:32:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 14:32:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:32:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.87 s +[02-19 14:32:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 14:32:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 14:32:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 14:32:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 14:32:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 14:32:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 14:32:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:32:41|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 14:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 14:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 30.86 s +[02-19 14:33:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:33:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:33:13|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 14:33:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/config.yaml +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:33:14|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="The video concludes with a final orbit around the statue, returning to a view similar to the opening shot but from a slightly different angle. The complete appreciation of the statue's artistry and its harmonious integration into the environment is emphasized, with the reflective surface consistently capturing the light and details of the surroundings. The final frames leave a lasting impression of the statue's dynamic presence and the serene campus setting." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.json +[02-19 14:33:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4 +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 14:33:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 14:33:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 14:33:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 14:46:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 14:46:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:46:40|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:46:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 14:46:41|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 14:46:43|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..c56d09c6ff6f72620b3647f58da64cde41d7cab3 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a close-up of the bronze horse statue, focusing on its rearing pose and detailed features like the mane and tail. The camera starts to pan, revealing the light beige pedestal with a plaque. The initial movement sets the stage for the exploration of the statue's reflective surface and the surrounding campus environment, which includes green lawns and palm trees.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..12dfc35a0d1c1e74bf8035144e216f14a5bd6aa7 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30d8469f3c18745a5d74c19719d59e45b1b734cef21b7a44b2dd221a3f2cedd9 +size 1785270 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.json b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..9a112aaf56f1970692f1856aae5b641c1f14c870 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues to move, it captures the statue from various angles, emphasizing its geometric integrity and the consistency of its shape. The background becomes more visible, showing modern buildings and pathways within the campus. The reflective bronze surface mirrors the environment, maintaining texture fidelity and providing a dynamic interplay of light and shadow.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..421e49a50c5026cda3b99f60118603e44d7f5ff9 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e78f5fb288e2ab3a2b3b1736911b78409d00c770d1008edd04e634e97c7ffa6 +size 1355724 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.json b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..074f7fdf25211e12a9b81da87f1bead1f81def85 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera further orbits around the statue, offering a comprehensive view of its form and the surrounding area. The focus remains on the statue's identity persistence, with the reflective surface continuously capturing the details of the environment. The background reveals more of the campus, including additional greenery and distant buildings, enhancing the sense of place.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fab0fcde420f315af8468a2d2f84d75c69ea6275 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9712663495cf5732f10aa173195b3b26cd726298cdef299893293e1e4b129b4f +size 1150460 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.json b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..b60d8cadf1bbf9a1259b9cf307c86ef3a44e485a --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the next phase, the camera completes its circular motion, bringing the viewer closer to the statue once again. The full context of the statue's placement within the campus is now evident, with the street, parked cars, and pedestrians visible in the distance. The reflective quality of the bronze continues to highlight the vibrant surroundings, ensuring a cohesive visual experience.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a8913daca9437fd797a7b6842dc293a34fb84ea3 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d59f514bfbd0918dfd5fc2e7901777afdf4c14a42b7c9e6a997218672852e6d +size 1244442 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.json b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..cfabf930f08d9bf2c83073ea899376f5e865ac6b --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with a final orbit around the statue, returning to a view similar to the opening shot but from a slightly different angle. The complete appreciation of the statue's artistry and its harmonious integration into the environment is emphasized, with the reflective surface consistently capturing the light and details of the surroundings. The final frames leave a lasting impression of the statue's dynamic presence and the serene campus setting.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..769b5a72edf52b4a15afaed94b03552480d7b294 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3ce9d02d7991c208b4f9d7031bf5060fb5f8708a183502bd7740f130879e9f1 +size 1352937 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b53af497d83a730ac2bfed1ebc2d354a596c4ff --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/console.log b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/console.log new file mode 100644 index 0000000000000000000000000000000000000000..53356e8cf8e8920a11024294189146517c751022 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/console.log @@ -0,0 +1,22 @@ +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 15:43:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 15:43:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 15:44:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/config.yaml +[02-19 15:44:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 15:44:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 15:44:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.json +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 15:44:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 15:57:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 15:57:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 15:57:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:57:55|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e9e1178f0e3d836504bb2b2a551f95d5764d4a33 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/debug.log @@ -0,0 +1,353 @@ +[02-19 15:43:44|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 15:43:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 15:43:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 15:43:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 15:43:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 15:43:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 15:43:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 15:43:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 15:43:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 15:43:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 15:43:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 15:43:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 15:43:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 15:43:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 15:43:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 15:43:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 15:43:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 15:43:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 15:43:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 15:43:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 15:43:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 15:43:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 15:43:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 15:43:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 15:43:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 15:43:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 15:43:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 15:43:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 15:43:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 15:43:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 15:43:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 15:43:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 15:43:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 15:43:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 15:43:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 15:43:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 15:43:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 15:43:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 15:43:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 15:43:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 15:43:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 15:43:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 15:43:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 15:43:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 15:43:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 15:43:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 0.94 s +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 15:43:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.61 s +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.62 s +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.62 s +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 15:44:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 15:44:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 15:44:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 15:44:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.84 s +[02-19 15:44:27|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 15:44:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 15:44:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 15:44:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/config.yaml +[02-19 15:44:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 15:44:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 15:44:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='The video concludes with a series of wide shots that encompass the entire Santa Cruz Surfing Museum and its surroundings. The camera captures the building from various angles, including the lighthouse tower, the entrance, and the solar panels on the roof. The final shots show the museum in relation to the parking lot, the grassy area, and the ocean, providing a comprehensive view of the location. The American flag continues to wave in the wind, and the sun creates lens flares as it shines brightly in the sky.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 15:44:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.json +[02-19 15:44:32|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 15:44:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 15:44:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4 +[02-19 15:44:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 15:44:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 15:44:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 15:44:34|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 15:57:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 15:57:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 15:57:53|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 15:57:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:57:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 15:57:55|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..9106c432cbb71d3718981177427c6a94b40fa635 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a panoramic view of the Santa Cruz Surfing Museum and its lighthouse, showcasing the red brick structure with white trim and the American flag flying high. The camera slowly pans around the building, capturing the details of the lighthouse tower and the surfboards displayed near the entrance. The initial shots are taken during bright daylight, with the sun creating lens flares as it shines directly into the camera.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1bf0561e97798b4bce0f2555d0ab17b43b6bf7c7 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6be610a0d32feb88fa3218f0b9af878bb057c7b908d04d66ebb6d2cb44844b84 +size 900750 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.json b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..a767b25c1170334041328d62e9a3d40ddf8e3f79 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the camera moves closer to the museum, focusing on the side of the building and the intricate brickwork. The lighthouse tower is shown from different angles, highlighting its white exterior and the black railing at the top. The camera then pans to the front of the museum, where the entrance with the 'Santa Cruz Surfing Museum' sign is clearly visible. The solar panels on the roof are also captured in detail.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b5adcfce1867a00565df3ebdfc0e26d72b4fc690 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d25ca157a9f7fb541a23d71b4fd87c523f50053183e6cbe02c3913d6c3a2e86 +size 954105 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.json b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..78d333036dc9d7482f30808006b1ae1e3e52716d --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The middle part of the video transitions to a broader view of the museum's surroundings. The camera captures the parking lot filled with cars and the grassy area adjacent to the building. The ocean is visible in the background, providing a scenic coastal backdrop. The video then focuses on the lighthouse tower once more, showing it from a higher angle and emphasizing its height and prominence.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..baf6e9b1ecf0cf6d11a124a83157b6025af90dff --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abd2128aa696efe67e88046deb670124548eb30c225a64467b702ec7725d95ed +size 1430603 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.json b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..1ab29043e63fb8c7f1b12b695f9a5790a718f7e1 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"In the latter part of the video, the camera moves around the museum, capturing the side and back views. The intricate details of the brick walls and the small windows with white frames are highlighted. The video then shifts to the front of the museum, where people are seen walking near the entrance. The American flag is shown waving in the breeze, and the ocean remains a constant presence in the background.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0c67b209a07ff2fd6430867f3156f95c9d9d1fc1 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2136b4d43db95b9b5681ae49d44cdffc304457c08c01a930aac6960e3b550ae +size 2733887 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.json b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..532b4cd07c449114227d213d57b0fce52ecafdd7 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with a series of wide shots that encompass the entire Santa Cruz Surfing Museum and its surroundings. The camera captures the building from various angles, including the lighthouse tower, the entrance, and the solar panels on the roof. The final shots show the museum in relation to the parking lot, the grassy area, and the ocean, providing a comprehensive view of the location. The American flag continues to wave in the wind, and the sun creates lens flares as it shines brightly in the sky.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cfb0b4356f70fe1bef04c22d10ef0ecdc45ec1b9 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:32a54962ace44fbd4a24447318fe4eeaeb2f9e327963114857c6b14595f472d2 +size 1510015 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..337149653d4d15c6f3b0e505b8d5501fa9f178eb --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/console.log b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..174bf6cced5a55b238ef89ce227c6e2985f25974 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/console.log @@ -0,0 +1,22 @@ +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:32:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:32:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/config.yaml +[02-19 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.json +[02-19 14:32:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:46:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 14:46:24|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8e172cef2f59d82c0a288e6b24f88f2fff3ccf66 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/debug.log @@ -0,0 +1,353 @@ +[02-19 14:32:09|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 14:32:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:32:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:32:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 14:32:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 14:32:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 14:32:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 14:32:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 14:32:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 14:32:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 14:32:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 14:32:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 14:32:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 14:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 14:32:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 14:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 14:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 14:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 14:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 14:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 14:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 14:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 14:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 14:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 14:32:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 14:32:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 14:32:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 14:32:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 14:32:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 14:32:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 14:32:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 14:32:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 14:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 14:32:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 14:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 14:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 14:32:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:32:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:32:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:32:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 14:32:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:32:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 14:32:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:32:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:32:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 14:32:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.00 s +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.64 s +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.64 s +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:32:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 14:32:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 14:32:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.84 s +[02-19 14:32:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:32:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:58|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 14:32:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/config.yaml +[02-19 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:32:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='The video concludes with a lingering shot of the statue, capturing the emotional depth of the scene and the peaceful ambiance of the campus environment. The final perspective offers a comprehensive view of the statue, highlighting both the front and back details, including the inscriptions on the base. The surrounding area is shown in its entirety, with the brick-paved courtyard, benches, and lush landscaping creating a harmonious setting.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.json +[02-19 14:32:59|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4 +[02-19 14:32:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 14:33:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 14:33:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 14:33:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 14:46:21|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:46:22|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 14:46:23|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 14:46:24|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..1296c69262b2b32949b0fa5402da039d52798d5d --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up of the bronze family statue, capturing the kneeling man, standing woman, and child in a moment of familial connection. The camera begins its slow orbit from the left side, revealing the circular courtyard with wooden benches, a tall palm tree, and neatly trimmed hedges. The initial angle emphasizes the statue's detailed drapery and the greenish patina on the bronze, while the background shows a building entrance with large glass windows.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6188fa6f4e473c9738b634a9af31a284b3e12867 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da500c919802c821bb731e4aff3e494c701fcb991dca0e6bc942bc80c555f670 +size 1716451 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.json b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..67ee80060e1f04beb18840e50777e8db6f4fa69f --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera moves to the right, it continues its orbit, showcasing the statue from different angles. The middle section focuses on the base of the statue, where the inscriptions 'IN CELEBRATION OF FAMILY' and 'Louis and Dorina Bruteau' are clearly visible. The background transitions to include more greenery, a pathway, and distant buildings with red-tiled roofs. The smooth camera movement allows for a continuous appreciation of the statue's texture and the serene campus setting.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c163795ff130c2b53644e9092aa613242ae4805b --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8c3753c81f832e89bb0356cdf28a4114e0882904126b956269766c394257228 +size 2151557 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.json b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..d77d456c0dd995511085be417c5d2f37bde2f2b9 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera continues its orbit, now showing the statue from the rear, highlighting the flowing robes and the tender pose of the figures. The background features a mix of architectural elements, including a building with large windows and a red-tiled roof, as well as more greenery and additional benches. The camera maintains a steady pace, ensuring that the viewer can fully appreciate the statue's form and the surrounding environment.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c51f220f566adc625004d0de975aa16f63729c14 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c2fcc3a1ae38792b5f8c047721b172d420980103d0dc69efdf4a234611eaee4 +size 1396574 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.json b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..c71be14e9cfc6fcb3d91c4b7bb3d992a54db302f --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"As the camera completes its orbit, it returns to the initial viewpoint near the building entrance. The final segment focuses on the front of the statue, capturing the interplay of light and shadow on its surface. The surrounding area is shown in its entirety, with the brick-paved courtyard, benches, and lush landscaping creating a harmonious setting. Pedestrians can be seen walking along the pathways in the distance, adding life to the tranquil scene.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7daed01a8c27fe70d33c41240fc90cc338a824c4 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c9f3a8911b2de975d647634dc8d7146561ec1a36ba2741c028b9db0c06b3468b +size 1115603 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.json b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..5517b2e8488e70dccce146c519b4e5ecb62cbf2f --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The video concludes with a lingering shot of the statue, capturing the emotional depth of the scene and the peaceful ambiance of the campus environment. The final perspective offers a comprehensive view of the statue, highlighting both the front and back details, including the inscriptions on the base. The surrounding area is shown in its entirety, with the brick-paved courtyard, benches, and lush landscaping creating a harmonious setting.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..de230b1a9ba8d49c09c1c220a3f878f443bc6f53 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4c8fee334d22ddb6d2993629f34b7919848cafafa65896312cc193ce4041e2a +size 1261131 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23918a4a9835c22bc4d4427a8b705689ef560353 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/console.log b/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/console.log new file mode 100644 index 0000000000000000000000000000000000000000..1d31d32a2ad86eb0e72cbbfba60f0569b306c92a --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/console.log @@ -0,0 +1,22 @@ +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 13:34:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 13:34:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 13:34:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 13:34:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 13:35:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/config.yaml +[02-19 13:35:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 13:35:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 13:35:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.json +[02-19 13:35:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 13:35:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 13:48:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 13:48:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 13:48:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 13:48:33|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b577a66277827169bae68c56915dba5266b3d0f2 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/debug.log @@ -0,0 +1,352 @@ +[02-19 13:34:23|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 13:34:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 13:34:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 13:34:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 13:34:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 13:34:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 13:34:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 13:34:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 13:34:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 13:34:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 13:34:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 13:34:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 13:34:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 13:34:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 13:34:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 13:34:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 13:34:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 13:34:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 13:34:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 13:34:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 13:34:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 13:34:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 13:34:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 13:34:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 13:34:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 13:34:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 13:34:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 13:34:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 13:34:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 13:34:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 13:34:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 13:34:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 13:34:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 13:34:35|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 13:34:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 13:34:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 13:34:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 13:34:35|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 13:34:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 13:34:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 13:34:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 13:34:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 13:34:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 13:34:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 13:34:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 13:34:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 13:34:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 13:34:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 13:34:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 13:34:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 13:34:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.11 s +[02-19 13:34:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 13:34:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 13:34:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 13:34:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 13:34:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.20 s +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.71 s +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.71 s +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.71 s +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 13:34:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 13:34:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 13:35:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 13:35:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 13:35:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.10 s +[02-19 13:35:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 13:35:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 13:35:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 13:35:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 13:35:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/config.yaml +[02-19 13:35:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 13:35:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 13:35:13|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt='The video starts with a view of the back rows of the auditorium, showcasing the beige upholstered seats with patterned fabric, wooden armrests, and seat bases arranged in curved rows on a light-colored carpeted floor. The camera then moves forward along the central aisle, revealing the stage area with a wooden podium, a large projection screen, and track lighting on the ceiling. Next, the perspective shifts to the side aisles, showing additional seating and the consistent design of the seats and the overall layout. The camera then returns to the front rows, providing a detailed view of the stage and the intricate ceiling lighting. Finally, the video concludes with a sweeping view of the entire auditorium, emphasizing the uniformity and professional aesthetic of the space.' negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 13:35:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.json +[02-19 13:35:13|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 13:35:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 13:35:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 13:35:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 13:35:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 13:35:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 13:48:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 13:48:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 13:48:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 13:48:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 13:48:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 13:48:33|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3faec517b01e3da11a3a1b6f5372771e3aa58f42 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a view of the back rows of the auditorium, showcasing the beige upholstered seats with patterned fabric, wooden armrests, and seat bases arranged in curved rows on a light-colored carpeted floor. The camera then moves forward along the central aisle, revealing the stage area with a wooden podium, a large projection screen, and track lighting on the ceiling. Next, the perspective shifts to the side aisles, showing additional seating and the consistent design of the seats and the overall layout. The camera then returns to the front rows, providing a detailed view of the stage and the intricate ceiling lighting. Finally, the video concludes with a sweeping view of the entire auditorium, emphasizing the uniformity and professional aesthetic of the space.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..88dc5f77d1bbd9aea3f77b60ce890366ba1f5ef4 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84b4f25cceef2ce7281248b87c16da284650671872e5dc370d5ee738f203ee2f +size 1605720 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9dd0eef7d20a37d3ab28542853c295c9094be60 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/console.log b/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/console.log new file mode 100644 index 0000000000000000000000000000000000000000..8bf69129f721431b0adb35721982803a28e47879 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/console.log @@ -0,0 +1,22 @@ +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:46:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:46:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/config.yaml +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.json +[02-19 14:46:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 15:00:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 15:00:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 15:00:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:00:17|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..cb52116b0c31aff8e49c73d726e4efffb16decc2 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/debug.log @@ -0,0 +1,352 @@ +[02-19 14:46:09|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 14:46:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:46:17|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:46:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 14:46:18|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 14:46:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 14:46:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 14:46:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 14:46:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 14:46:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 14:46:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 14:46:18|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 14:46:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 14:46:18|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 14:46:19|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 14:46:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 14:46:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 14:46:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 14:46:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 14:46:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 14:46:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 14:46:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 14:46:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 14:46:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 14:46:19|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 14:46:20|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 14:46:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 14:46:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 14:46:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 14:46:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 14:46:21|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 14:46:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 14:46:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 14:46:22|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 14:46:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 14:46:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:46:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:46:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 14:46:22|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:46:22|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:46:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:46:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 14:46:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:46:24|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 14:46:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:46:24|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:46:25|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.66 s +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.67 s +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.67 s +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:46:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:46:26|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:46:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 14:46:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 14:46:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 27.84 s +[02-19 14:46:51|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:46:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:56|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:46:57|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/config.yaml +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 14:46:57|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video starts with a close-up view of the playground's main wooden structure, showcasing its detailed design with brown posts, silver slides, and black swings. The teal spring riders are prominently displayed in the foreground. As the camera pans to the left, it captures the light gray benches and orange/yellow trash bins, emphasizing the playground's amenities. Moving further, the background reveals a residential area with houses, a red fence, and tall trees, providing context to the playground's location. The camera continues its smooth rotation, offering a comprehensive view of the entire playground, ensuring that the structural geometry and surface textures of all equipment remain consistent and accurately depicted." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.json +[02-19 14:46:57|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 14:46:57|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 14:46:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 14:46:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 15:00:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 15:00:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 15:00:15|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 15:00:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:00:16|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 15:00:17|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..4dc48b28f56acd9368f5ea6c2025ac1f2ca6eb19 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video starts with a close-up view of the playground's main wooden structure, showcasing its detailed design with brown posts, silver slides, and black swings. The teal spring riders are prominently displayed in the foreground. As the camera pans to the left, it captures the light gray benches and orange/yellow trash bins, emphasizing the playground's amenities. Moving further, the background reveals a residential area with houses, a red fence, and tall trees, providing context to the playground's location. The camera continues its smooth rotation, offering a comprehensive view of the entire playground, ensuring that the structural geometry and surface textures of all equipment remain consistent and accurately depicted.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..edff2d4f7a403a220489e4c55efb7525bc2542da --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e113b7fb6d58854120c1033c011d28a2678470a8c6f306d28ed35271c0000aad +size 2132357 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4dd4922f0875aa492292fdc61098ffdac4ee550 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/console.log b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/console.log new file mode 100644 index 0000000000000000000000000000000000000000..32e2b633b775c0dd439cd7718d8786d5f95fb89a --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/console.log @@ -0,0 +1,22 @@ +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:31:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:31:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:32:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/config.yaml +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.json +[02-19 14:32:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:45:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:45:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:45:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 14:45:51|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..15bf5c42de3b98fc769e4fa6c58f57ef4b2654c4 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/debug.log @@ -0,0 +1,353 @@ +[02-19 14:31:43|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 14:31:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:31:51|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:31:52|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 14:31:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 14:31:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 14:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 14:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 14:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 14:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 14:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 14:31:53|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 14:31:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 14:31:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 14:31:53|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 14:31:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 14:31:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 14:31:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 14:31:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 14:31:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 14:31:53|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 14:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 14:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 14:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 14:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 14:31:54|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 14:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 14:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 14:31:55|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 14:31:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 14:31:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 14:31:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 14:31:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 14:31:56|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 14:31:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 14:31:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:31:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:31:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 14:31:56|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:31:56|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:31:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:31:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 14:31:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:31:58|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 14:31:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:31:58|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.13 s +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 14:31:59|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.16 s +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.67 s +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.68 s +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.68 s +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 14:32:00|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 14:32:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 14:32:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:32:01|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 26.93 s +[02-19 14:32:25|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:32:30|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:30|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:31|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/config.yaml +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:32:31|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="In the final segment, the camera provides a detailed view of the loader's front and side, capturing the intricate details of the bucket, tires, and cabin. The vibrant yellow paint, accented with black and red, is highlighted against the backdrop of a clear blue sky. The CAT logo and model number '950F' are prominently displayed, along with the O Nelson & Son Construction sticker. The loader remains stationary, allowing for a thorough examination of its design and build quality. The surrounding environment of dry grass, scattered trees, and a clear blue sky provides a natural and serene setting for the machine." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.json +[02-19 14:32:31|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:32:31|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4 +[02-19 14:32:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 14:32:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 14:32:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 14:32:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 14:45:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 14:45:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:45:49|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:45:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 14:45:50|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 14:45:51|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..9958f1aa9387c857a8927bbe93a16f78c2ad2a50 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video begins with a close-up shot of the yellow Caterpillar 950F wheel loader positioned in a dry, grassy field. The camera starts by focusing on the front of the machine, capturing the large bucket with its sharp teeth and the massive front tires. The loader's vibrant yellow paint, accented with black and red, is highlighted against the backdrop of a clear blue sky. The CAT logo and model number '950F' are visible on the side, along with the O Nelson & Son Construction sticker. The loader remains stationary as the camera begins to pan around it.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8ddb1cf7c007398797c2f7fe98627fdfab516b1a --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e9f2ffbe8a4bdd122e61a7e822b5c87f547eb5483f0e04dc84c35cf864eb52df +size 1936627 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.json b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..44e9dab9e9641e2e0bdc460de0588c855a0cd30a --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues to move, it reveals the side profile of the Caterpillar 950F, showcasing the large cabin with its expansive windows and the dual rear tires with deep treads. The smooth panning motion allows for a detailed view of the machine's structure and components. The O Nelson & Son Construction sticker remains in view, providing contact information. The surrounding environment of dry grass and trees is consistently visible, with the clear blue sky overhead. The loader begins a slow rotation, preparing to reveal more of its features.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d0b4c886ae5d319eeca8f829624f646e605e8bf8 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7d8e61a20f07ea6077ebcc733ce7ab92a0fd95c913d9c4d718a211d3ff5b9eaa +size 1721032 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.json b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..df3888b1d004ca039669cf93ed6e99ab1c419fd2 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"The camera now focuses on the rear section of the loader as it continues its rotation. The massive dual rear tires and the Caterpillar-branded cabin are prominently displayed, with the large front bucket still visible from the side. The O Nelson & Son Construction sticker is still present, adding a touch of branding. The loader's slow movement allows for a comprehensive view of its rear and side profiles. The background of dry grass, scattered trees, and a clear blue sky remains consistent, providing a natural setting for the machine.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d0e418de7d56b199863a3dae423c530a0c8734ab --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9311c32732195f2ecab164146e65cba95bdc75e7b5459dc0d2c98640b6b1eeb8 +size 1305539 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.json b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..588aad6ea0a79452699d3ef97a988572516e8a7f --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"As the loader completes its rotation, the camera captures the front of the machine once again, emphasizing the large bucket and the massive front tires. The CAT logo and model number '950F' are clearly visible, reinforcing the loader's identity. The O Nelson & Son Construction sticker is still in view, providing contact details. The surrounding environment of dry grass, scattered trees, and a clear blue sky remains consistent, highlighting the loader's presence in the field. The camera movement is smooth, ensuring all key features are captured from various angles.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..87e1ceb18002fd654be6bfe6031100042d15ddae --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27ad2a3e27df1ba4114e1d08b8aed1b55017edb6989b251d39ca610f0ad1d184 +size 1137951 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.json b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..24dca6051f192e341ba47d19f9b399558e80f5e7 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final segment, the camera provides a detailed view of the loader's front and side, capturing the intricate details of the bucket, tires, and cabin. The vibrant yellow paint, accented with black and red, is highlighted against the backdrop of a clear blue sky. The CAT logo and model number '950F' are prominently displayed, along with the O Nelson & Son Construction sticker. The loader remains stationary, allowing for a thorough examination of its design and build quality. The surrounding environment of dry grass, scattered trees, and a clear blue sky provides a natural and serene setting for the machine.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3d2d44fc1578272502cffdc6fd314e2c471f8609 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f457205fb6099e9f752a7caed2e47e565fe2cae411c343c7d6477d7107c394b4 +size 1009373 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3bdbcd5b13ec1f53993173a98dac62dd4175d9b0 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/console.log b/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/console.log new file mode 100644 index 0000000000000000000000000000000000000000..356c0617138fc015bfc47008a4e2232a8735bb6a --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/console.log @@ -0,0 +1,22 @@ +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:47:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:47:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/config.yaml +[02-19 14:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 14:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 14:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.json +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 15:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 15:01:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 15:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:01:09|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a63cbbc5e7966668907486e0d37f6b31de95462b --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/debug.log @@ -0,0 +1,352 @@ +[02-19 14:46:53|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 14:47:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:47:00|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:47:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 14:47:02|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 14:47:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 14:47:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 14:47:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 14:47:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 14:47:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 14:47:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 14:47:02|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 14:47:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 14:47:02|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 14:47:03|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 14:47:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 14:47:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 14:47:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 14:47:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 14:47:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 14:47:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 14:47:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 14:47:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 14:47:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 14:47:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 14:47:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 14:47:04|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 14:47:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 14:47:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 14:47:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 14:47:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 14:47:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 14:47:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 14:47:06|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 14:47:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 14:47:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:47:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:47:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 14:47:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:47:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:47:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:47:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 14:47:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:47:07|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 14:47:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:47:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:47:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 14:47:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:47:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-19 14:47:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 14:47:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 14:47:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 14:47:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:47:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.64 s +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.65 s +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.65 s +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:47:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:47:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:47:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 14:47:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 14:47:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.37 s +[02-19 14:47:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:47:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:47:42|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 14:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/config.yaml +[02-19 14:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_0'] +[02-19 14:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_0 +[02-19 14:47:42|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_0' prompt_path=None prompt="The video opens with a view of the beige, single-story building's side, emphasizing the white lattice paneling on the garage, the beige horizontal siding, and the dark gray shingled roof with two skylights. The camera then begins to move, gradually revealing the front of the building, which includes an open pavilion area with interior fixtures and two white circular picnic tables with built-in benches. As the camera continues its movement, it captures the back and side sections of the building, maintaining a focus on the consistent identity of the structure, including its brown wooden trim and the sharp texture of the siding and roof. The video concludes with a final orbit around the building, providing a comprehensive view of all its sections and features." negative_prompt='' seed=42 guidance=7 inference_type= input_path=None resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 14:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.json +[02-19 14:47:42|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:47:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:47:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 14:47:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 14:47:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 14:47:43|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 15:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 15:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 15:01:06|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 15:01:06|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:01:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 15:01:09|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..3a5462464116ef543102b1d8c7472a3a9ada1919 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a view of the beige, single-story building's side, emphasizing the white lattice paneling on the garage, the beige horizontal siding, and the dark gray shingled roof with two skylights. The camera then begins to move, gradually revealing the front of the building, which includes an open pavilion area with interior fixtures and two white circular picnic tables with built-in benches. As the camera continues its movement, it captures the back and side sections of the building, maintaining a focus on the consistent identity of the structure, including its brown wooden trim and the sharp texture of the siding and roof. The video concludes with a final orbit around the building, providing a comprehensive view of all its sections and features.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c02c29ce07f3f34650581f1278c4ba2b0d89a88c --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7fed9e55b4e6f058474a0f054087679fb5a1fc6ccb1ad57797cd0247e30c855 +size 1660429 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97de46ca3d956072347dc27df2ad7146a5336ba7 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/console.log b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/console.log new file mode 100644 index 0000000000000000000000000000000000000000..60d344fa0bb91e09229e269879ca25f0103e0820 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/console.log @@ -0,0 +1,22 @@ +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:32:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:32:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:32:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/config.yaml +[02-19 14:32:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:32:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:32:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.json +[02-19 14:32:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:32:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:46:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 14:46:10|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f66300c0307045a47d4a4e0333ebab6c95091203 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/debug.log @@ -0,0 +1,353 @@ +[02-19 14:31:59|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 14:32:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:32:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:32:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 14:32:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 14:32:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 14:32:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 14:32:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 14:32:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 14:32:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 14:32:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 14:32:09|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 14:32:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 14:32:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 14:32:09|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 14:32:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 14:32:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 14:32:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 14:32:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 14:32:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 14:32:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 14:32:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 14:32:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 14:32:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 14:32:10|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 14:32:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 14:32:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 14:32:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 14:32:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 14:32:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 14:32:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 14:32:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 14:32:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 14:32:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 14:32:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 14:32:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:32:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:32:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 14:32:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:32:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:32:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:32:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 14:32:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:32:14|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 14:32:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:32:14|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 14:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 0.95 s +[02-19 14:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 14:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 14:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 14:32:15|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:32:15|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.16 s +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.68 s +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.68 s +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.68 s +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 14:32:16|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 14:32:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 14:32:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:32:17|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:32:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 14:32:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 14:32:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.21 s +[02-19 14:32:42|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:32:47|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:47|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:32:48|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 14:32:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:32:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/config.yaml +[02-19 14:32:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:32:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:32:48|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt="The final segment focuses on the monument from a high angle, capturing the interplay of light and shadow on its surface. The camera then moves to a lower angle, providing a comprehensive view of the entire monument against the sky. The video concludes with a wide shot of the monument in its entirety, framed by the park's greenery and pathways, under varying lighting conditions." negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 14:32:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.json +[02-19 14:32:48|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:32:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:32:48|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4 +[02-19 14:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 14:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 14:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 14:32:49|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:46:08|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:46:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 14:46:09|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 14:46:10|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..f17cb2c8971a83bb17de9c2ab9df3195b2c33961 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a close-up of the monument's base, focusing on the white marble steps and the inscription panel. The camera slowly tilts upward, revealing the four fluted columns and the bronze seated statue within the open-air structure. The initial frames capture the monument from a low angle, emphasizing its grandeur and the detailed craftsmanship of the marble.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f0c82d18ab6850850ad3c765c6914578b476ade1 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:229bb8c5d8d52ab0ae12f0108d5600ecaebb9d9aa5cbdb9c0b8e9df375e246c4 +size 748364 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.json b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..9909b8423f5b1abce1ef21f1632750d8c3adfec1 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the video progresses, the camera begins to pan around the monument, providing a 360-degree view of its design. The middle frames highlight the decorative sculptural elements at the top, including the spire and winged figures, while also showcasing the surrounding park environment with its lush greenery and pathways. The lighting remains consistent, with a mix of overcast and sunlit conditions.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..382305d0a799f1b7ce984fa0ff9d6a46077cbe25 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed904fab60f6274047ec2706cba3e5015c48fa77a5fcb4ff54362f23087d9752 +size 1445405 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.json b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..fb5e20c2894254fe50799b8ee7e3bb48510f34b3 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"In the next segment, the camera continues to circle the monument, focusing on the upper sections and the intricate details of the sculptures. The frames capture the monument from various angles, emphasizing its geometric integrity and the texture of the marble. The background remains a serene park setting, with trees and pathways visible throughout.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..112b07b959561840b50230a6f4fe5f943636cba5 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a8128418c40cdb4720fac8c619d566c7b9280d18eac5f4f1dcb8f18be701a8e +size 1019674 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.json b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..440b268f2eee54d463ceacaefc40f64c0f095f24 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"Toward the end of this segment, the camera ascends to capture the full height of the monument, showcasing the spire and winged figures against the sky. The lighting shifts to a brighter, sunlit condition, highlighting the monument's details and creating dynamic shadows. The surrounding park environment remains a constant backdrop, adding to the overall composition.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..44119834d1b585a8188a25785d9ea8a3c8476f50 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3c2faa28a4f264e5a03f6ede04daf0bb1f66125259373739cb16ae34f902ad5 +size 1093183 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.json b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..45829227760023324b85b6f92f672d2385be9192 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"The final segment focuses on the monument from a high angle, capturing the interplay of light and shadow on its surface. The camera then moves to a lower angle, providing a comprehensive view of the entire monument against the sky. The video concludes with a wide shot of the monument in its entirety, framed by the park's greenery and pathways, under varying lighting conditions.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..eb32f6fef9e1be98b389eb09f262d050a070b130 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:844e8777330e9d67397ea26820c12a5c156a83b37e61adf5723470759b9dfee6 +size 1122837 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e8bc9017eae547bd8590be6e082e73f1749d0602 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/console.log b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/console.log new file mode 100644 index 0000000000000000000000000000000000000000..15adf4a8c4565bd7c4ebb9f4403375b43bbeef34 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/console.log @@ -0,0 +1,22 @@ +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:36:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:36:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:36:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/config.yaml +[02-19 14:36:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:36:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:36:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_4.json +[02-19 14:36:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:36:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:51:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:51:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:51:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 14:51:08|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..069b9995bc7b0b30234670c3012ab4eed37b1e44 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/debug.log @@ -0,0 +1,353 @@ +[02-19 14:35:56|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 14:36:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:36:03|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 14:36:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 14:36:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 14:36:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 14:36:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 14:36:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 14:36:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 14:36:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 14:36:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 14:36:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 14:36:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 14:36:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 14:36:05|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 14:36:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 14:36:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 14:36:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 14:36:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 14:36:05|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 14:36:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 14:36:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 14:36:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 14:36:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 14:36:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 14:36:06|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 14:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 14:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 14:36:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 14:36:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 14:36:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 14:36:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 14:36:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 14:36:08|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 14:36:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 14:36:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 14:36:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 14:36:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 14:36:08|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 14:36:08|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 14:36:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:36:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 14:36:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:36:10|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 14:36:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 14:36:10|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.09 s +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 14:36:11|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 14:36:11|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.13 s +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.63 s +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.63 s +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.63 s +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 14:36:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 14:36:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 14:36:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 14:36:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 28.60 s +[02-19 14:36:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 14:36:43|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:43|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 14:36:44|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 14:36:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 14:36:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/config.yaml +[02-19 14:36:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 14:36:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 14:36:44|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='80-100%: Toward the end of the video, the camera provides a comprehensive overview of the ballroom, moving smoothly from one section to another. The ornate white marble fireplace is once again a focal point, with the camera circling around it to showcase its full grandeur. The various tables and chairs are shown from multiple perspectives, emphasizing their arrangement and the overall aesthetic of the space. The large arched windows with heavy beige curtains are captured from a higher angle, highlighting their symmetry and the natural light they bring into the room. The intricate coffered ceiling is revisited, with the camera panning across it to capture the full expanse of its geometric patterns. The video concludes with a final wide-angle shot, encapsulating the elegance and sophistication of the grand ballroom.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 14:36:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_4.json +[02-19 14:36:44|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 14:36:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 14:36:44|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_3.mp4 +[02-19 14:36:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 14:36:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 14:36:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 14:36:45|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 14:51:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 14:51:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 14:51:05|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 14:51:05|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 14:51:07|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 14:51:08|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..45f14c210d4524ba90421776eefeb197453cf18f --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"0-20%: The video opens with a panoramic shot of the grand ballroom, capturing the stunning coffered ceiling with its alternating light and dark geometric patterns. The modern spherical pendant lights with petal-like shades are evenly spaced, casting a warm glow over the room. The dark wood paneling on the walls and the large arched windows draped with heavy beige curtains are immediately noticeable, with soft natural light filtering through. The ornate white marble fireplace stands out at the far end, adding to the room's elegance. Various tables, including round, S-shaped, and long banquet styles, along with wooden chairs (some with black seats), are arranged throughout the space, creating a sense of order and sophistication.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9246876d0aef20cfd5f09eefda74a5c91e843e0d --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad201c5008d461cbad8acfea5d8eb4a2d03a976386657b868c2577134289803e +size 780972 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_1.json b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..cb98e226d4a683727d52c70306e422bb90d81507 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"20-40%: The camera begins to pan across the ballroom, revealing more details of the environment. The intricate wood paneling on the walls is highlighted, showcasing its vertical grooves and rich texture. The patterned carpet with circular motifs in brown and gold is clearly visible, adding warmth to the space. The spherical pendant lights continue to provide a soft, ambient light, enhancing the room's inviting atmosphere. The large arched windows with heavy beige curtains are shown from different angles, emphasizing their size and the natural light they allow into the room. The ornate white marble fireplace remains a central feature, with its decorative reliefs catching the light.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_1.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6a1bcb16b54a7432d67df4709332c498de156f7b --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c3a574bed610477e7f383a87c11d4a5be6fa0b13a6f5814fb50e315cc7c9f20 +size 1339131 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_2.json b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..979f2edebc4c5b7d8997fd8c7614095f69ea298b --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"40-60%: As the camera continues to move, it focuses on different sections of the ballroom, providing closer views of the furniture and architectural details. The round and S-shaped tables are shown in greater detail, highlighting their sleek designs and the contrast between the white tabletops and the dark wooden frames. The wooden chairs with black seats are arranged neatly around the tables, contributing to the room's organized appearance. The camera then shifts to the walls, capturing the interplay of light and shadow created by the pendant lights and the intricate wood paneling. The large arched windows are shown again, this time from a lower angle, emphasizing their height and the way they frame the natural light.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_2.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2bd55c1cb80359bc1b71e0dd6b76d0724c0c77a8 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:155489eba73d01a50b972e8127f6d56bc1bd492bc0211f4c1263a91ecd8ee02b +size 1083965 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_3.json b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..0b9f93a00c30ca06ecae396ec23812229ee40aab --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"60-80%: The video transitions to a series of close-up shots, focusing on specific elements of the ballroom. The ornate white marble fireplace is examined in detail, with its decorative reliefs and the warm light reflecting off its surface. The camera then moves to the large arched windows, capturing the way the heavy beige curtains are drawn back to reveal the bright daylight outside. The intricate coffered ceiling is shown from different angles, highlighting the precision of its geometric patterns and the alternating light and dark colors. The spherical pendant lights are also revisited, with the camera capturing their unique petal-like shades and the way they cast light throughout the room.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_3.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b2f6e3dd40f24ef5db38c77a87e8eccf5b155530 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5ba9663cadeb01840ba27d02a1c29e79b1b699764da6da17a190d8e4a6aa93b5 +size 1288300 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_4.json b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..55481ee405ff9c644e49952ae2c71933e67cbecb --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"80-100%: Toward the end of the video, the camera provides a comprehensive overview of the ballroom, moving smoothly from one section to another. The ornate white marble fireplace is once again a focal point, with the camera circling around it to showcase its full grandeur. The various tables and chairs are shown from multiple perspectives, emphasizing their arrangement and the overall aesthetic of the space. The large arched windows with heavy beige curtains are captured from a higher angle, highlighting their symmetry and the natural light they bring into the room. The intricate coffered ceiling is revisited, with the camera panning across it to capture the full expanse of its geometric patterns. The video concludes with a final wide-angle shot, encapsulating the elegance and sophistication of the grand ballroom.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_4.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..26862b68e4f0aa515bcda8e0811d6b21ba787b71 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c01e1cdb0bb3db97380cf015f475a7ec3442d09d6e5272208b2768ce57c6d8b +size 2364031 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/config.yaml b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b816a57b7d61c7a4f021837c411fb1e4b51ee2ad --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/config.yaml @@ -0,0 +1,760 @@ +checkpoint: + broadcast_via_filesystem: 'False' + dcp_allow_mismatched_size: 'False' + dcp_async_mode_enabled: 'False' + enable_gcs_patch_in_boto3: 'False' + jit: + device: cuda + dtype: bfloat16 + enabled: 'False' + input_shape: null + strict: 'True' + keys_not_to_resume: [] + load_ema_to_reg: 'False' + load_from_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + load_path: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt + load_training_state: 'False' + only_load_scheduler_state: 'False' + save_iter: '1000' + save_to_object_store: + bucket: bucket + credentials: credentials/s3_checkpoint.secret + enabled: 'True' + strict_resume: 'True' + type: + _target_: + callbacks: null + disable_async: 'False' + verbose: 'True' +dataloader_train: + dataloaders: + image_data: + dataloader: + batch_size: '12' + cache_augment_fn: functools.partial(, + n=1) + cache_size: '8' + concat_size: '1' + dataset: + augmentor_name: image_basic_augmentor_without_embeddings + caption_type: qwen2p5_7b_v4 + dataset_resolution_type: gt720p + embedding_type: null + resolution: '720' + num_workers: '6' + use_cache: 'False' + ratio: '1' + video_data: + dataloader: + batch_size: '1' + cache_augment_fn: functools.partial(, n=1.8) + cache_size: '16' + concat_size: '1' + dataset: + augmentor_name: video_basic_augmentor_v2 + caption_type: t2w_qwen2p5_7b + dataset_resolution_type: all + embedding_type: null + max_fps_thres: '60' + min_fps_thres: '10' + num_video_frames: '93' + resolution: '720' + use_native_fps: 'True' + video_decoder_name: video_naive_bytes + use_cache: 'False' + ratio: '3' +dataloader_val: + _target_: + dataloaders: + image_data: + dataloader: + _target_: + batch_size: '2' + cache_augment_fn: null + cache_replay_name: image_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' + video_data: + dataloader: + _target_: + batch_size: '1' + cache_augment_fn: null + cache_replay_name: video_dataloader + cache_size: '32' + concat_size: '1' + dataset: + _target_: + len_t5: '512' + num_video_frames: '136' + resolution: '512' + t5_dim: '1024' + num_workers: '8' + pin_memory: 'True' + shuffle: 'False' + use_cache: 'False' + webdataset: 'False' + ratio: '1' +defaults: +- _self_ +- data_train: mock +- data_val: mock +- optimizer: fusedadamw +- scheduler: lambdalinear +- model: ddp +- callbacks: basic +- net: null +- conditioner: video_prediction_conditioner +- ema: power +- tokenizer: cosmos_tokenizer_causal_cv8x8x8_c16_res720_t121_it121_v1_0 +- checkpoint: s3 +- ckpt_type: dummy +- experiment: null +job: + cluster: null + group: official_runs_vid2vid + name: Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt + project: cosmos_diffusion_v2 + wandb_mode: online +model: + _recursive_: 'False' + _target_: + config: + conditional_frame_timestep: 0.1 + conditional_frames_probs: + 0: 0.5 + 1: 0.25 + 2: 0.25 + conditioner: + _target_: + fps: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: fps + output_key: fps + padding_mask: + _target_: + dropout_rate: '0.0' + dtype: null + input_key: padding_mask + output_key: padding_mask + text: + _target_: + credential_path: credentials/s3_training.secret + dropout_rate: '0.2' + empty_string_embeddings_path: s3://bucket/predict2_assets/reason1_empty_string_embeddings.pt + input_key: + - t5_text_embeddings + use_empty_string: 'False' + use_video_condition: + _target_: + dropout_rate: '0.0' + input_key: fps + output_key: use_video_condition + conditioning_strategy: frame_replace + denoise_replace_gt_frames: true + ema: + enabled: false + iteration_shift: 0 + rate: 0.1 + fsdp_shard_size: 1 + high_sigma_ratio: 0.05 + high_sigma_timesteps_max: 1000 + high_sigma_timesteps_min: 980 + init_lora_weights: true + input_caption_key: ai_caption + input_data_key: video + input_image_key: images + lora_alpha: 32 + lora_rank: 32 + lora_target_modules: q_proj,k_proj,v_proj,output_proj,mlp.layer1,mlp.layer2 + max_num_conditional_frames: 2 + min_num_conditional_frames: 0 + net: + _target_: + adaln_lora_dim: '256' + atten_backend: minimal_a2a + concat_padding_mask: 'True' + crossattn_emb_channels: '1024' + crossattn_proj_in_channels: '100352' + extra_per_block_abs_pos_emb: 'False' + in_channels: '16' + max_frames: '128' + max_img_h: '240' + max_img_w: '240' + model_channels: '2048' + num_blocks: '28' + num_heads: '16' + out_channels: '16' + patch_spatial: '2' + patch_temporal: '1' + pos_emb_cls: rope3d + pos_emb_interpolation: crop + pos_emb_learnable: 'True' + rope_enable_fps_modulation: 'False' + rope_h_extrapolation_ratio: '3.0' + rope_t_extrapolation_ratio: '1.0' + rope_w_extrapolation_ratio: '3.0' + sac_config: + every_n_blocks: 1 + mode: predict2_2b_720_aggressive + timestep_scale: '0.001' + use_adaln_lora: 'True' + use_crossattn_projection: 'True' + use_wan_fp32_strategy: 'True' + precision: bfloat16 + resolution: '720' + shift: 5 + state_ch: 16 + state_t: 24 + text_encoder_class: reason1p1_7B + text_encoder_config: + ckpt_path: s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/ + compute_online: true + embedding_concat_strategy: full_concat + model_config: + _target_: + model_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenModelConfig + activation_checkpoint: + mode: selective + models: vlm + selective_ac_option: op + add_answer_tag: 'True' + add_cross_attention: 'False' + add_image_start_end_tag: 'False' + add_tile_tag: 'False' + architectures: + - Qwen2_5_VLForConditionalGeneration + attention_dropout: '0.0' + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + aux_loss_coeff: '0.0' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: '151643' + cache_dir: null + checkpoint: + async_mode: disabled + create_seed_checkpoint: false + enable_checkpoint: false + export_dtype: float32 + folder: checkpoint + interval: 500 + interval_type: steps + model_weights_only: false + chunk_size_feed_forward: '0' + ckpt_dir: null + ckpt_path: null + comm: + init_timeout_seconds: 300 + trace_buf_size: 20000 + train_timeout_seconds: 100 + cp_size: null + cross_attention_hidden_size: null + decoder_start_token_id: null + deterministic: 'False' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + encoder_no_repeat_ngram_size: '0' + eos_token_id: '151645' + ep_size: null + experimental: + enable_async_tensor_parallel: false + enable_compiled_autograd: false + pipeline_parallel_degree: 1 + exponential_decay_length_penalty: null + finetuning_task: null + float8: + enable_float8_linear: false + forced_bos_token_id: null + forced_eos_token_id: null + freeze_llm: 'False' + freeze_mm_projector: 'False' + freeze_vision_encoder: 'False' + fsdp_enabled: 'False' + hidden_act: silu + hidden_size: '3584' + id2label: + 0: LABEL_0 + 1: LABEL_1 + image_token_id: '151655' + initializer_range: '0.02' + intermediate_size: '18944' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + loss_per_token: 'True' + max_batch_size: '1' + max_length: '20' + max_position_embeddings: '128000' + max_seq_len: '128000' + max_window_layers: '28' + min_length: '0' + mm_projector: null + model_type: qwen2_5_vl + name_or_path: Qwen/Qwen2.5-VL-7B-Instruct + no_repeat_ngram_size: '0' + num_attention_heads: '28' + num_beam_groups: '1' + num_beams: '1' + num_hidden_layers: '28' + num_key_value_heads: '4' + num_return_sequences: '1' + num_tiles: '1' + optimizer: + early_step_in_backward: false + end_lr: 2.5e-05 + fused: false + init_lr: 1.0e-05 + lr: 0.0003 + lr_multiplier_llm: 1.0 + lr_multiplier_mm_projector: 1.0 + lr_multiplier_vision_encoder: 0.1 + name: AdamW + output_attentions: 'False' + output_hidden_states: 'True' + output_scores: 'False' + pad_token_id: null + precision: bfloat16 + prefix: null + prepend_padding: 'False' + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + rms_norm_eps: 1e-06 + rope_scaling: + mrope_section: + - '16' + - '24' + - '24' + rope_type: default + type: default + rope_theta: '1000000.0' + s3_credential_path: credentials/pbss_dir.secret + seed: '0' + sep_token_id: null + sliding_window: '32768' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'False' + tile_tag_type: space_separated + tokenizer_class: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + training: + compile: false + context_parallel_degree: 1 + data_parallel_replicate_degree: 1 + data_parallel_shard_degree: -1 + disable_loss_parallel: false + enable_cpu_offload: false + fsdp_reshard_after_forward: default + mixed_precision_param: bfloat16 + mixed_precision_reduce: float32 + steps: 400000 + tensor_parallel_degree: 1 + use_cosine_decay: false + use_linear_decay: true + warmup_steps: 1000 + training_seq_len: '4096' + transformers_version: 4.51.0.dev0 + typical_p: '1.0' + use_bfloat16: 'False' + use_cache: 'False' + use_fsdp2: 'True' + use_return_dict: 'True' + use_rope_from_torchtitan: 'False' + use_sliding_window: 'False' + video_token_id: '151656' + vision_config: + _target_: cosmos_predict2._src.reason1.configs.default.model_config_qwen.QwenVisionConfig + add_cross_attention: 'False' + architectures: null + attn_implementation: flash_attention_2 + attn_implementation_autoset: 'True' + bad_words_ids: null + begin_suppress_tokens: null + bos_token_id: null + chunk_size_feed_forward: '0' + cross_attention_hidden_size: null + decoder_start_token_id: null + depth: '32' + diversity_penalty: '0.0' + do_sample: 'False' + early_stopping: 'False' + embed_dim: null + encoder_no_repeat_ngram_size: '0' + eos_token_id: null + exponential_decay_length_penalty: null + finetuning_task: null + forced_bos_token_id: null + forced_eos_token_id: null + fullatt_block_indexes: + - '7' + - '15' + - '23' + - '31' + hidden_act: silu + hidden_size: '1280' + id2label: + 0: LABEL_0 + 1: LABEL_1 + in_channels: '3' + in_chans: '3' + intermediate_size: '3420' + is_decoder: 'False' + is_encoder_decoder: 'False' + label2id: + LABEL_0: '0' + LABEL_1: '1' + length_penalty: '1.0' + max_length: '20' + min_length: '0' + mlp_ratio: null + model_type: qwen2_5_vl + name_or_path: '' + no_repeat_ngram_size: '0' + num_beam_groups: '1' + num_beams: '1' + num_heads: '16' + num_return_sequences: '1' + out_hidden_size: '3584' + output_attentions: 'False' + output_hidden_states: 'False' + output_scores: 'False' + pad_token_id: null + patch_size: '14' + prefix: null + problem_type: null + pruned_heads: _Nothing.NOTHING + remove_invalid_values: 'False' + repetition_penalty: '1.0' + return_dict: 'True' + return_dict_in_generate: 'False' + sep_token_id: null + spatial_merge_size: '2' + spatial_patch_size: '14' + suppress_tokens: null + task_specific_params: null + temperature: '1.0' + temporal_patch_size: '2' + tf_legacy_loss: 'False' + tie_encoder_decoder: 'False' + tie_word_embeddings: 'True' + tokenizer_class: null + tokens_per_second: '2' + top_k: '50' + top_p: '1.0' + torch_dtype: bfloat16 + torchscript: 'False' + typical_p: '1.0' + use_bfloat16: 'False' + window_size: '112' + vision_encoder: openai/clip-vit-base-patch32 + vision_encoder_config: + depth_init: true + dim: 1024 + ffn_dim_multiplier: null + head_dim: null + hidden_act: null + hidden_dim: 4096 + image_size: 1024 + image_token_id: null + multiple_of: null + n_heads: 16 + n_kv_heads: null + n_layers: 24 + norm_eps: 1.0e-05 + norm_type: rmsnorm + num_channels: 3 + patch_size: 16 + proj_bias: null + qkv_bias: null + rope_theta: 10000.0 + use_cache: false + use_rope_from_torchtitan: false + vision_encoder_in_channels: '3' + vision_end_token_id: '151653' + vision_start_token_id: '151652' + vision_token_id: '151654' + vocab_size: '152064' + z_loss_coeff: '0.0' + tokenizer: + _target_: + cache_dir: null + tokenizer_type: Qwen/Qwen2.5-VL-7B-Instruct + n_layers_per_group: 5 + s3_credential_path: credentials/s3_checkpoint.secret + tokenizer: + _target_: + chunk_duration: '81' + load_mean_std: 'False' + name: wan2pt1_tokenizer + temporal_window: '16' + train_time_distribution: logitnormal + train_time_weight: reweighting + use_dora: false + use_dynamic_shift: false + use_high_sigma_strategy: false + use_kerras_sigma_at_inference: true + use_lora: false + use_torch_compile: false +model_parallel: + _cpu_offloading_context: null + async_tensor_model_parallel_allreduce: false + autocast_dtype: torch.float32 + barrier_with_L1_time: true + batch_p2p_comm: true + batch_p2p_sync: true + bf16: false + context_parallel_size: 2 + cpu_offloading: false + cpu_offloading_activations: true + cpu_offloading_double_buffering: false + cpu_offloading_num_layers: 0 + cpu_offloading_weights: true + cross_entropy_fusion_impl: native + cross_entropy_loss_fusion: false + deallocate_pipeline_outputs: false + defer_embedding_wgrad_compute: false + delay_wgrad_compute: false + deterministic_mode: false + enable_autocast: false + expert_model_parallel_size: 1 + expert_tensor_parallel_size: 1 + finalize_model_grads_func: null + fp16: false + grad_scale_func: null + grad_sync_func: null + gradient_accumulation_fusion: false + hierarchical_context_parallel_sizes: null + microbatch_group_size_per_vp_stage: 1 + moe_extended_tp: false + no_sync_func: null + num_microbatches_with_partial_activation_checkpoints: null + overlap_moe_expert_parallel_comm: false + overlap_p2p_comm: false + overlap_p2p_comm_warmup_flush: false + param_sync_func: null + params_dtype: torch.float32 + perform_initialization: true + pipeline_dtype: null + pipeline_model_parallel_comm_backend: null + pipeline_model_parallel_size: 1 + sequence_parallel: false + tensor_model_parallel_size: 1 + timers: null + tp_comm_atomic_ag: false + tp_comm_atomic_rs: false + tp_comm_bootstrap_backend: nccl + tp_comm_bulk_dgrad: true + tp_comm_bulk_wgrad: true + tp_comm_overlap: false + tp_comm_overlap_ag: true + tp_comm_overlap_disable_fc1: false + tp_comm_overlap_disable_qkv: false + tp_comm_overlap_rs: true + tp_comm_overlap_rs_dgrad: false + tp_comm_split_ag: true + tp_comm_split_rs: true + use_cpu_initialization: false + use_ring_exchange_p2p: false + use_te_rng_tracker: false + variable_seq_lengths: false + virtual_pipeline_model_parallel_size: null + wgrad_deferral_limit: 0 +optimizer: + _target_: + betas: + - '0.9' + - '0.999' + eps: 1e-08 + fused: 'True' + lr: 3e-05 + model: null + optim_type: adamw + weight_decay: '0.001' +scheduler: + _target_: + cycle_lengths: + - '400000' + f_max: + - '0.99' + f_min: + - '0.4' + f_start: + - 1e-06 + verbosity_interval: '0' + warm_up_steps: + - '100' +trainer: + callbacks: + compile_tokenizer: + _target_: + compile_after_iterations: '4' + dynamic: 'False' + enabled: 'True' + dataloader_speed: + _target_: + every_n: '200' + save_s3: 'True' + step_size: '1' + device_monitor: + _target_: + every_n: '200' + log_memory_detail: 'True' + save_s3: 'True' + step_size: '1' + upload_every_n_mul: '10' + every_n_sample_ema: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'True' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + every_n_sample_reg: + _target_: + do_x0_prediction: 'False' + every_n: '1000000000000' + fps: '16' + guidance: + - '0.0' + - '3.0' + - '7.0' + is_ema: 'False' + n_sample_to_save: '128' + n_sigmas_for_x0_prediction: '4' + n_viz_sample: '3' + num_sampling_step: '35' + prompt_type: t5_xxl + run_at_start: 'False' + save_s3: 'True' + step_size: '1' + use_negative_prompt: 'False' + grad_clip: + _target_: + clip_norm: '0.1' + force_finite: 'True' + heart_beat: + _target_: + every_n: '10' + save_s3: 'True' + step_size: '1' + update_interval_in_minute: '20' + iter_speed: + _target_: + every_n: '200' + hit_thres: '5' + save_s3: 'True' + save_s3_every_log_n: '10' + low_prec: + _target_: + config: null + trainer: null + update_iter: '1' + manual_gc: + _target_: + every_n: '200' + warm_up: '5' + wandb: + _target_: + logging_iter_multipler: '1' + save_logging_iter_multipler: '10' + save_s3: 'True' + wandb_10x: + _target_: + logging_iter_multipler: '10' + save_logging_iter_multipler: '1' + save_s3: 'True' + cudnn: + benchmark: 'True' + deterministic: 'False' + ddp: + broadcast_buffers: 'True' + find_unused_parameters: 'False' + static_graph: 'True' + distributed_parallelism: fsdp + grad_accum_iter: '1' + grad_scaler_args: + enabled: 'False' + logging_iter: '200' + max_iter: '150000' + max_val_iter: null + memory_format: torch.preserve_format + profiling: + enable_memory_snapshot: 'False' + enable_profiling: 'False' + profile_freq: '1' + profile_memory: 'False' + record_shape: 'False' + save_s3: 'False' + target_ranks: + - '0' + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + with_modules: 'True' + with_stack: 'True' + run_validation: 'False' + run_validation_on_start: 'False' + seed: '0' + straggler_detection: + analyze_backward: 'True' + analyze_dataloading: 'True' + analyze_forward: 'True' + analyze_optimizer: 'True' + enabled: 'True' + max_diff: '1.5' + profile_freq: '1' + raise_error: 'True' + report_freq: '100' + timeout_period: '999999999' + type: + validation_iter: '100' +upload_reproducible_setup: 'True' diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/console.log b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/console.log new file mode 100644 index 0000000000000000000000000000000000000000..c12eb457013219672cc1dc46913964de6154b41b --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/console.log @@ -0,0 +1,22 @@ +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 15:43:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 15:43:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 15:44:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/config.yaml +[02-19 15:44:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 15:44:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 15:44:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.json +[02-19 15:44:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 15:44:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 15:57:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 15:57:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 15:57:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:57:35|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/debug.log b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..1dcc098ccab5d3f8e5cda029d048c80f569ede2f --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/debug.log @@ -0,0 +1,353 @@ +[02-19 15:43:23|DEBUG|generation/pipelines/cosmos-predict2.5/packages/cosmos-oss/cosmos_oss/init.py:176:init_output_dir] Flags(internal=False, smoke=False, verbose=False, experimental_checkpoints=False) +[02-19 15:43:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 15:43:30|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/env_parsers/env_parser.py:86:get_val_dict] getting val dict of CredentialEnvParser +[02-19 15:43:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/visualize/video.py:31:] No module named 'ffmpegcv' +[02-19 15:43:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/checkpointer/dcp.py:188:] for the back comptiable pytorch! New DefaultLoadPlanner class is created. +[02-19 15:43:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:32:__init__] SetupArguments(output_dir=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1') model='2B/post-trained' checkpoint_path='/home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt' experiment='Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt' config_file='cosmos_predict2/_src/predict2/configs/video2world/config.py' context_parallel_size=1 offload_diffusion_model=False offload_tokenizer=False offload_text_encoder=False disable_guardrails=True offload_guardrail_models=True keep_going=True profile=False) +[02-19 15:43:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v1... +[02-19 15:43:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering video_basic_augmentor_v2... +[02-19 15:43:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering noframedrop_nocameramove_video_augmentor_v1... +[02-19 15:43:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering nocameramove_video_augmentor_v1... +[02-19 15:43:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor... +[02-19 15:43:32|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/datasets/augmentor_provider.py:94:augmentor_register] registering image_basic_augmentor_without_embeddings... +[02-19 15:43:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] Flash Attention v3 is not supported because the Python package was not found. +[02-19 15:43:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/attention/utils/safe_log.py:35:debug] NATTEN Attention is not supported due to insufficient NATTEN version natten.__version__='0.21.0', expected at least NATTEN_MIN_RELEASE_VERSION=[0, 21, 5], or NATTEN_MIN_DEV_VERSION=([0, 21, 5], 9). +[02-19 15:43:32|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2._src.predict2.configs.video2world.experiment +[02-19 15:43:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings +[02-19 15:43:32|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1 +[02-19 15:43:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_14b_reason_1p1_rectified_flow +[02-19 15:43:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1 +[02-19 15:43:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_2B_reason_1p1_rectified_flow +[02-19 15:43:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.model_7b_reason_1p1 +[02-19 15:43:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_14B_index_3 +[02-19 15:43:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.reason_embeddings.stage3_2B +[02-19 15:43:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world +[02-19 15:43:33|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_14B +[02-19 15:43:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.sparse_2B +[02-19 15:43:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3 +[02-19 15:43:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.stage3_2B_ablation +[02-19 15:43:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.resume_text2world.state3_14B_index_3 +[02-19 15:43:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model +[02-19 15:43:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_14B_RF +[02-19 15:43:35|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B +[02-19 15:43:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2._src.predict2.configs.video2world.experiment.specialized_model.SFT_2B_RF +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_only2 +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_only2 +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_only2 +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_only2 +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_only2 +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_only2 +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_only2 +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_high_motion_high_sigma +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_av_high_sigma +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_physical_ai_high_sigma +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_4k_cooldown_high_sigma +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_face_high_sigma2 +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_crowded_high_sigma2 +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-1-Size-2B-Res-720-Fps-16-Note-rf_robotics_high_sigma2 +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/configs/video2world/experiment/specialized_model/SFT_2B_RF.py:853:] Storing Stage-c_pt_4-Index-2-Size-2B-Res-720-Fps-16-Note-rf_with_edm_ckpt +[02-19 15:43:36|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:192:import_all_modules_from_package] Reloading all modules from package cosmos_predict2.experiments +[02-19 15:43:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base +[02-19 15:43:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.action +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/post-trained(81edfebe-bd6a-4039-8c1d-737df1a790bf) +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt'} +[02-19 15:43:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-2B/base/pre-trained(d20b7120-df3e-4911-919d-db6e08bad31c) +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '15a82a2ec231bc318692aa0456a36537c806e7d4', 'filename': 'base/pre-trained/d20b7120-df3e-4911-919d-db6e08bad31c_ema_bf16.pt'} +[02-19 15:43:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.cosmos_nemo_assets_lora +[02-19 15:43:36|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/config_helper.py:210:import_modules_recursively] Reloading module cosmos_predict2.experiments.base.groot +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Predict2.5-14B/base/pre-trained(54937b8c-29de-4f04-862c-e67b04ec41e8) +[02-19 15:43:36|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-14B', 'repo_type': 'model', 'revision': '03eb354f35eae0d6e0c1be3c9f94d8551e125570', 'filename': 'base/pre-trained/54937b8c-29de-4f04-862c-e67b04ec41e8_ema_bf16.pt'} +[02-19 15:43:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:91:load_model_from_checkpoint] Overriding config checkpoint path with: /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 15:43:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/misc.py:152:set_random_seed] Using random seed 0. +[02-19 15:43:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:111:load_model_from_checkpoint] Loading model from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 15:43:37|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:132:__init__] DiffusionModel: precision torch.bfloat16 +[02-19 15:43:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Wan2.1/vae(685afcaa-4de2-42fe-b7b9-69f7a2dee4d8) +[02-19 15:43:37|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:99:_download] Downloading checkpoint file from Hugging Face with {'repo_id': 'nvidia/Cosmos-Predict2.5-2B', 'repo_type': 'model', 'revision': '6787e176dce74a101d922174a95dba29fa5f0c55', 'filename': 'tokenizer.pth'} +[02-19 15:43:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/tokenizers/wan2pt1.py:669:_video_vae] loading /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Predict2.5-2B/snapshots/6787e176dce74a101d922174a95dba29fa5f0c55/tokenizer.pth +[02-19 15:43:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 15:43:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_tokenizer: 1.05 s +[02-19 15:43:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #0-fps: + ReMapkey + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: fps + Dtype: None +[02-19 15:43:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #1-padding_mask: + ReMapkey + input key: padding_mask + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: padding_mask + Dtype: None +[02-19 15:43:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #2-text: + TextAttr + input key: ['t5_text_embeddings'] + Param count: 0 + Trainable: None + Dropout rate: 0.2 + Output key: [crossattn_emb] +[02-19 15:43:38|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/conditioner.py:434:__init__] Initialized embedder #3-use_video_condition: + BooleanFlag + input key: fps + Param count: 0 + Trainable: None + Dropout rate: 0.0 + Output key: use_video_condition + This is a boolean flag +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1624:build_pos_embed] Building positional embedding with rope3d class, impl +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:886:__init__] Using AdaLN LoRA Flag: True. We enable bias if no AdaLN LoRA for backward compatibility. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:38|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is None and using 16 heads with a dimension of 128. +[02-19 15:43:39|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:432:__init__] Setting up Attention. Query dim is 2048, context_dim is 1024 and using 16 heads with a dimension of 128. +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1804:enable_selective_checkpoint] Enable selective checkpoint with predict2_2b_720_aggressive, for every 1 blocks. Total blocks: 28 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 0 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 1 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 2 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 3 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 4 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 5 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 6 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 7 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 8 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 9 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 10 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 11 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 12 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 13 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 14 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 15 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 16 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 17 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 18 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 19 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 20 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 21 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 22 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 23 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 24 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 25 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 26 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/networks/minimal_v4_dit.py:1810:enable_selective_checkpoint] Enable selective checkpoint for block 27 +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on meta to cuda and broadcast model states: 0.14 s +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model: 0.67 s +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on Creating PyTorch model and ema if enabled: 0.68 s +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on DiffusionModel: set_up_model: 0.68 s +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:74:__init__] Instantiating text encoder model... +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint Qwen/Qwen2.5-VL-7B-Instruct(7219c6c7-f878-4137-bbdb-76842ea85e70) +[02-19 15:43:39|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/tokenizer/processor.py:75:__init__] Successfully loaded processor from local cache +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:116:__init__] Setting torch default dtype from torch.float32 to torch.bfloat16 +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:50:_validate] dp_shard is set to -1, will be automatically determined based on world_size 1 // 1. +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:54:_validate] dp_shard is set to 1. +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:75:build_mesh] Building 0-D device mesh with [], [] +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallel_dims.py:105:build_mesh] mesh: DeviceMesh('cuda', 0) +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/parallelisms/parallelize_qwen.py:336:apply_ac] Applied selective activation checkpointing to the model +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/reason1/models/vlm_base.py:123:__init__] Reset torch default dtype to torch.float32 +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:84:__init__] Loading checkpoint from s3://bucket/cosmos_reasoning1/sft_exp700/sft_exp721-1_qwen7b_tl_721_5vs5_s3_balanced_n32_resume_16k/checkpoints/iter_000016000/model/. +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:181:path] Downloading checkpoint nvidia/Cosmos-Reason1.1-7B(cb3e3ffa-7b08-4c34-822d-61c7aa31a14f) +[02-19 15:43:40|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpoint_db.py:134:_download] Downloading checkpoint from Hugging Face with {'repo_id': 'nvidia/Cosmos-Reason1-7B', 'repo_type': 'model', 'revision': '3210bec0495fdc7a8d3dbb8d58da5711eab4b423', 'allow_patterns': ['*']} +[02-19 15:44:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:113:__init__] Finished loading checkpoint from /home/bravo/.cache/huggingface/hub/models--nvidia--Cosmos-Reason1-7B/snapshots/3210bec0495fdc7a8d3dbb8d58da5711eab4b423. +[02-19 15:44:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/text_encoders/text_encoder.py:116:__init__] Text encoder model instantiated +[02-19 15:44:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/timer.py:150:_log] Time spent on instantiate model: 29.31 s +[02-19 15:44:07|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/utils/model_loader.py:227:load_model_state_dict_from_checkpoint] Loading model cached locally from /home/bravo/zzhangg/MemBench/generation/pipelines/cosmos-predict2.5/models/Cosmos-Predict2.5-2B/base/post-trained/81edfebe-bd6a-4039-8c1d-737df1a790bf_ema_bf16.pt +[02-19 15:44:11|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:919:load_state_dict] load model in non-strict mode +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.0.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.1.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.2.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.3.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.4.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.5.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.6.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.7.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.8.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.9.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.10.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.11.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.12.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.13.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.14.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.15.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.16.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.17.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.18.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.19.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.20.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.21.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.22.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.23.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.24.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.25.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.26.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.self_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.q_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key blocks.27.cross_attn.k_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:11|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/checkpointer.py:451:non_strict_load_model] Skipping key t_embedding_norm._extra_state introduced by TransformerEngine for FP8 in the checkpoint. +[02-19 15:44:12|CRITICAL|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/models/text2world_model_rectified_flow.py:920:load_state_dict] [RANK 0] _IncompatibleKeys(missing_keys=[], unexpected_keys=[], incorrect_shapes=[]) +[02-19 15:44:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/distributed.py:415:sync_model_states] Synchronizing model states from rank 0 to all ranks in process group [0]. +[02-19 15:44:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:60:__init__] Saved config to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/config.yaml +[02-19 15:44:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:82:generate] Generating 1 samples: ['segment_4'] +[02-19 15:44:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:86:generate] [1/1] Processing sample segment_4 +[02-19 15:44:12|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:93:_generate_sample] InferenceArguments(name='segment_4' prompt_path=None prompt='In the final 20%, the camera completes its full orbit around the truck, returning to the initial side view. The video concludes by reiterating the key visual features of the truck, such as the rusted paint, wooden bed, and branding, while also showcasing the consistency of the urban setting throughout the entire sequence.' negative_prompt='' seed=46 guidance=7 inference_type= input_path=PosixPath('/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4') resolution='none' num_output_frames=77 num_steps=35 enable_autoregressive=True chunk_size=77 chunk_overlap=1) +[02-19 15:44:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:99:_generate_sample] Saved arguments to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.json +[02-19 15:44:12|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:113:_generate_sample] Guardrail checks on prompt are disabled +[02-19 15:44:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:118:_generate_sample] Generating video with autoregressive mode... +[02-19 15:44:12|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:730:generate_autoregressive_from_batch] Processing video input for autoregressive: /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4 +[02-19 15:44:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:798:generate_autoregressive_from_batch] Generating 1 chunks with chunk_size=77, chunk_overlap=1 for 77 total frames +[02-19 15:44:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:816:generate_autoregressive_from_batch] Processing chunk 1/1, frames 0-77 +[02-19 15:44:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:556:generate_vid2world] GPU memory usage after getting data_batch: 19.79 GB +[02-19 15:44:13|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:594:generate_vid2world] [Memory Optimization] Starting latent sample generation +[02-19 15:57:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/predict2/inference/video2world.py:878:generate_autoregressive_from_batch] Generated final video with shape torch.Size([1, 3, 77, 704, 1280]) +[02-19 15:57:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:134:_generate_sample] Autoregressive generation completed, preparing to save video... +[02-19 15:57:33|WARNING|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:172:_generate_sample] Guardrail checks on video are disabled +[02-19 15:57:33|INFO|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:175:_generate_sample] Saving video to %s.mp4 ... +[02-19 15:57:34|DEBUG|generation/pipelines/cosmos-predict2.5/cosmos_predict2/_src/imaginaire/utils/easy_io/handlers/imageio_video_handler.py:163:dump_to_fileobj] mimsave_kwargs: {'fps': 16, 'quality': 5, 'macro_block_size': 1, 'ffmpeg_params': ['-s', '1280x704'], 'output_params': ['-f', 'mp4']} +[02-19 15:57:35|SUCCESS|generation/pipelines/cosmos-predict2.5/cosmos_predict2/inference.py:178:_generate_sample] Saved video to /home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.mp4 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.json b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.json new file mode 100644 index 0000000000000000000000000000000000000000..94f82ecf205d3c79fa2628f47fbc6b07b53ede30 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.json @@ -0,0 +1 @@ +{"name":"segment_0","prompt_path":null,"prompt":"The video opens with a side view of the vintage blue flatbed truck, focusing on the wooden slats of the bed and the rusted blue paint. The camera begins its movement by panning left, gradually revealing the front of the truck and its weathered grille. The initial 20% of the video sets the stage by establishing the truck's presence in the urban environment, with trees and outdoor seating visible in the background.","negative_prompt":"","seed":42,"guidance":7,"inference_type":"text2world","input_path":null,"resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f81475a85452be4dc67339c0972d688cd31cef2a --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00ba42864612505237a56509c233024f813c78de16f1aba6e8698df8abd6ff09 +size 1330290 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.json b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.json new file mode 100644 index 0000000000000000000000000000000000000000..37322d373396f0b1185b3decfd2e657ffe34753d --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.json @@ -0,0 +1 @@ +{"name":"segment_1","prompt_path":null,"prompt":"As the camera continues its orbit, it moves past the front of the truck, capturing the details of the headlights and the 'SAN PEDRO SQUARE MARKET' logo on the door. The next 20% of the video focuses on the transition from the front to the rear, with the camera tilting slightly upward to emphasize the height of the wooden bed. The background remains consistent, featuring the same urban elements but from different angles.","negative_prompt":"","seed":43,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_0.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..08251ddd2ebb20b83105254a7536fd109f5b7e1d --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d3d9b81518c95652775cd63caf389840e80aa2535d0857662a709f8319171b5a +size 1688507 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.json b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.json new file mode 100644 index 0000000000000000000000000000000000000000..f92b1cd965265fa3e4e95fb2ee35f23803eda84a --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.json @@ -0,0 +1 @@ +{"name":"segment_2","prompt_path":null,"prompt":"In the third segment, the camera reaches the rear of the truck, providing a clear view of the tailgate, license plate, and red tail lights. The surrounding area includes a glimpse of the street and additional buildings, adding depth to the scene. The camera then begins its return journey toward the front, maintaining a steady pace and keeping the truck centered in the frame.","negative_prompt":"","seed":44,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_1.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..01ef6fa6b4a40a925f6b3f9702c50e65b1a199af --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:efcf8ac23e27ea9de6b84d282c9f84ccbee0b3e771f9b6e97dc2781fdf9eedb7 +size 1903033 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.json b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.json new file mode 100644 index 0000000000000000000000000000000000000000..ea809cbd63bdfc0020a1cc2c44d7bb83b3c93632 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.json @@ -0,0 +1 @@ +{"name":"segment_3","prompt_path":null,"prompt":"The fourth segment brings the camera back to the front of the truck, where it lingers for a moment to highlight the details of the grille and the overall condition of the vehicle. The camera then pans slightly to the right, offering a three-quarter view that combines both the front and side perspectives. This part of the video reinforces the truck's vintage appeal and its integration into the urban landscape.","negative_prompt":"","seed":45,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_2.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..6846b704438953146b30f06628afdc3bceee0bd8 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f4cd012978ef9141a211187dd3491e8dfb82d8e1512ad3167d729f77e908a5d +size 1167141 diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.json b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.json new file mode 100644 index 0000000000000000000000000000000000000000..cf29d17d4abb6fe0b4e8a8e1ce23d2db0a1abf2a --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.json @@ -0,0 +1 @@ +{"name":"segment_4","prompt_path":null,"prompt":"In the final 20%, the camera completes its full orbit around the truck, returning to the initial side view. The video concludes by reiterating the key visual features of the truck, such as the rusted paint, wooden bed, and branding, while also showcasing the consistency of the urban setting throughout the entire sequence.","negative_prompt":"","seed":46,"guidance":7,"inference_type":"video2world","input_path":"/home/bravo/zzhangg/MemBench/generation/outputs/cosmos/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_3.mp4","resolution":"none","num_output_frames":77,"num_steps":35,"enable_autoregressive":true,"chunk_size":77,"chunk_overlap":1} \ No newline at end of file diff --git a/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.mp4 b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4955a4ff445f3b0076935226361ab4a55da9d637 --- /dev/null +++ b/0219133414_cosmos_tnt_object/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1/segment_4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad4bdac0148d62c84310679c728266430853cc0eb243a7e236c1513a9f096d8a +size 1687490 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..066cc8c17176237d243c8c064dcf61606a48bdab --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_0b199d304a74f9c3d2003c36f68bbf95fa52d9983fd2275bdae66a61e23d49d5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:107399d3a1b253e5f6db097f0c7e3b6fe5f9dfc78bb5f80b517f767c8a7f69d9 +size 7186498 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..74b45239e3d3bb2bfcc1ccddac2ac21f27a1a3f9 --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_15d42cee734662bf8471fa3ead0266b2e0ffc015a980be20d4aee17b727c70e0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e17dcb22d51981e0e2743f0cb8ce5311a531d05c8fd415745c926d1a8144c9bd +size 7859789 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c1cd7cd01343d239887d7ce4b7dba4fba2d0a220 --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_1a5e108074f33f006cf0feeaf343ca72253f8fe7be2d43c0a97e7c71026294f1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ea923628fc01aca82cfe84d99d79543ae022e7eb213490dd4fa31fa189424ad +size 1872960 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..77ff0c0483cd515107755c0a6d7439f7f93393ca --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_244802e503a003b212b2da2e7e2265cc15a97b19ec1933487096f94417132017.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:16a959ffa6137f68622a2eb45e68a4152225f5e839fd6dd97970f31d4c614342 +size 5840408 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..23a7cf36b2e7a35126deb098d53142dd012ed783 --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_347b192573971cc541124018fe08d0198d80de2b7133ef1be6bf93a51771450e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c50a8b74a4b242c7f60dbc8ff8c8186d75efefb5e25df80eef9c9105e2bd39c +size 6885672 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..679c58d840d38cd5032de76c5badce4b82699db9 --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_4917553f77bf46830cbc3ef9a6f6aea706f13c0938c5c828ae6670bf79045857.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c82513529ed9f380b9f1f31fba8de81c26ca0b5656478e8ebb72ad6e7522b5b6 +size 7526307 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..018b9708a1e04a7eb2dc96cc16f6e936d453c629 --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_5c230d78880b8c91f83954320759a4299ec025506ae677331c26e6220c455d3f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c83ab2c93bbe49846ae1246a8545cb185ac7c9a13e16de8cebb1649e50f90f5 +size 7638193 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..88dc5f77d1bbd9aea3f77b60ce890366ba1f5ef4 --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_6893f40dcebdc2fdddd025f89a8cfc754e4fbf4ce16d161f17af92a103b60bfc.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84b4f25cceef2ce7281248b87c16da284650671872e5dc370d5ee738f203ee2f +size 1605720 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..edff2d4f7a403a220489e4c55efb7525bc2542da --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_79b74782cc704f9e6608549e44015e34d5768e8cc047db690a788b827e84f6de.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e113b7fb6d58854120c1033c011d28a2678470a8c6f306d28ed35271c0000aad +size 2132357 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9d0f6b72f5ca41b92536144c81445d3da212f59 --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_88d372e8fdd3bf9ff3f3165e627d8e178998fc2481c48402178bda80a087ad21.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9ffbc8491820ae736a54387b303cc48d410a73914c2eb73f4e52e21aada5259 +size 7107365 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..c02c29ce07f3f34650581f1278c4ba2b0d89a88c --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_99869de4c05e3f558d74b51a7ffed9b1aab2f51cfc1e302f4be78756ea77b85b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7fed9e55b4e6f058474a0f054087679fb5a1fc6ccb1ad57797cd0247e30c855 +size 1660429 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7c5f262ba16d0a9586ffff4e050f7b7e3fdcacfe --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_a4764f2ff7a08c7d8dae83b3891516592f4b7eb732e6e4e645a235b8aa04138f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c53421b4acb3afd2d32de3055ec2b0ac877721dfc1edbbf95b6b66a9cad58a9 +size 5426306 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..60a853df2a72d2633470054cafd2aac0c28cee6e --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_a5630370d7039c78cc822dbd17a967711be3632bd2cdb43519eab8c2fa876144.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99187956a548bf06167dd42ee29836000d826473928fd89be0deffc47189196a +size 6853238 diff --git a/0219133414_cosmos_tnt_object/videos/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.mp4 b/0219133414_cosmos_tnt_object/videos/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e15de82c0206110bc314bbef62caae816ceaa977 --- /dev/null +++ b/0219133414_cosmos_tnt_object/videos/mem_tnt_c377e6c6722ada96fe40ab147f9fee05963e9cc40b9e6cdc51dd48e788f2e0d1.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f850dc27c3c654d905fbe26508f5a5a29785ef8ea1e6e2263ce8a3425c79198 +size 7773354